FA1DAC05FE9882…18B119726331

Arguments · 24

1. #1datastore
2. #2README.md
3. #3# Datastore Package Package provides support to store multiple collections of records. It supports the definition of multiple storages, where each one is a collection of records. Records can have any number of user defined fields which are added dynamically when values are set on a record. These fields can also be renamed or removed. Storages have support for simple schemas that allow users to pre-define fields which can optionally have a default value also defined. Default values are assigned to new records on creation. User defined schemas can optionally be strict, which means that records from a storage using the schema can only assign values to the pre-defined set of fields. In which case, assigning a value to an unknown field results in an error. Package also supports the definition of custom record indexes. Indexes are used by storages to search and iterate records. The default index is the ID index but custom single and multi value indexes can be defined. > [!WARNING] > Using this package to store your realm data must be carefully considered. > The fact that record fields are not strictly typed and can be renamed or removed > could lead to issues if not careful when coding your realm(s). So it's recommended > that you consider other alternatives first, like alternative patterns or solutions > provided by the blockchain to deal with data, types and data migration for example. Repository can be found at [jeronimoalbi/gnome](https://github.com/jeronimoalbi/gnome), as part of `jeronimoalbi`'s Gno smart contracts monorepo. ## Usage [embedmd]:# (filetests/readme_filetest.gno go) ```go package main import "gno.land/p/g17khqpukees4237dtn3astzapmp462vjhsz6st4/datastore" func main() { db := datastore.NewDatastore() // Define a unique case insensitive index for user emails emailIdx := datastore.NewIndex("email", func(r datastore.Record) string { return r.MustGet("email").(string) }).Unique().CaseInsensitive() // Create a new storage for user records users := db.CreateStorage("users", datastore.WithIndex(emailIdx)) // Add a user with a single "email" field user := users.NewRecord() user.Set("email", "foo@bar.org") // Save to assign the user ID and update indexes user.Save() // Find user by email using the custom index user, _ = users.Get(emailIdx.Name(), "FOO@bar.org") println("Found by email:", user.MustGet("email")) // Find user by ID user, _ = users.GetByID(user.ID()) println("Found by ID:", user.ID()) // Delete the user from the storage and update indexes users.Delete(user.ID()) println("Users:", users.Size()) } // Output: // Found by email: foo@bar.org // Found by ID: 1 // Users: 0 ``` ### Querying [embedmd]:# (filetests/readme_query_filetest.gno go) ```go package main import "gno.land/p/g17khqpukees4237dtn3astzapmp462vjhsz6st4/datastore" func main() { db := datastore.NewDatastore() // Define a multi value index to search posts by tag tagsIdx := datastore.NewMultiValueIndex("tags", func(r datastore.Record) []string { return r.MustGet("tags").([]string) }) posts := db.CreateStorage("posts", datastore.WithIndex(tagsIdx)) for _, title := range []string{"Post 1", "Post 2", "Post 3"} { post := posts.NewRecord() post.Set("title", title) post.Set("tags", []string{"gno", title}) post.Save() } // Get two records starting from the second one recordset, err := posts.Query(datastore.WithOffset(1), datastore.WithSize(2)) if err != nil { panic(err) } recordset.Iterate(func(r datastore.Record) bool { println("Post:", r.MustGet("title")) return false // Continue iterating }) // Get all the records that have the "gno" tag using the custom index recordset = posts.MustQuery(datastore.UseIndex("tags", "gno")) println("Tagged posts:", recordset.Size()) } // Output: // Post: Post 2 // Post: Post 3 // Tagged posts: 3 ```
4. #4datastore.gno
5. #5// Package datastore provides support to store multiple collections of records. package datastore import ( "errors" "gno.land/p/nt/bptree/v0" ) // ErrStorageExists indicates that a storage exists with the same name. var ErrStorageExists = errors.New("a storage with the same name exists") // NewDatastore creates a new store. func NewDatastore() *Datastore { return &Datastore{ storages: bptree.NewBPTree32(), } } // Datastore is a store that can contain multiple named storages. // A storage is a collection of records. // // Example usage: // // // Create an empty storage to store user records // var db Datastore // storage := db.CreateStorage("users") // // // Get a storage that has been created before // storage = db.GetStorage("profiles") type Datastore struct { storages *bptree.BPTree // string(name) -> *Storage } // CreateStorage creates a new named storage within the data store. func (ds *Datastore) CreateStorage(name string, options ...StorageOption) *Storage { if ds.storages.Has(name) { return nil } s := NewStorage(name, options...) ds.storages.Set(name, &s) return &s } // HasStorage checks if data store contains a storage with a specific name. func (ds *Datastore) HasStorage(name string) bool { return ds.storages.Has(name) } // GetStorage returns a storage that has been created with a specific name. // It returns nil when a storage with the specified name is not found. func (ds *Datastore) GetStorage(name string) *Storage { if s, ok := ds.storages.Get(name).(*Storage); ok { return s } return nil }
6. #6gnomod.toml
7. #7module = "gno.land/p/g17khqpukees4237dtn3astzapmp462vjhsz6st4/datastore" gno = "0.9"
8. #8index.gno
9. #9package datastore import ( "gno.land/p/moul/collection/v0" ) // DefaultIndexOptions defines the default options for new indexes. const DefaultIndexOptions = collection.DefaultIndex | collection.SparseIndex type ( // IndexFn defines a type for single value indexing functions. // This type of function extracts a single string value from // a record that is then used to index it. IndexFn func(Record) string // IndexMultiValueFn defines a type for multi value indexing functions. // This type of function extracts multiple string values from a // record that are then used to index it. IndexMultiValueFn func(Record) []string // Index defines a type for custom user defined storage indexes. // Storages are by default indexed by the auto geneated record ID // but can additionally be indexed by other custom record fields. Index struct { name string options collection.IndexOption fn interface{} } ) // NewIndex creates a new single value index. // // Usage example: // // // Index a User record by email // idx := NewIndex("email", func(r Record) string { // return r.MustGet("email").(string) // }) func NewIndex(name string, fn IndexFn) Index { return Index{ name: name, options: DefaultIndexOptions, fn: func(v interface{}) string { return fn(v.(Record)) }, } } // NewMultiValueIndex creates a new multi value index. // // Usage example: // // // Index a Post record by tag // idx := NewMultiValueIndex("tag", func(r Record) []string { // return r.MustGet("tags").([]string) // }) func NewMultiValueIndex(name string, fn IndexMultiValueFn) Index { return Index{ name: name, options: DefaultIndexOptions, fn: func(v interface{}) []string { return fn(v.(Record)) }, } } // Name returns index's name. func (idx Index) Name() string { return idx.name } // Options returns current index options. // These options define the index behavior regarding case sensitivity and uniquenes. func (idx Index) Options() collection.IndexOption { return idx.options } // Func returns the function that storage collections apply // to each record to get the value to use for indexing it. func (idx Index) Func() interface{} { return idx.fn } // Unique returns a copy of the index that indexes record values as unique values. // Returned index contains previous options plus the unique one. func (idx Index) Unique() Index { if idx.options&collection.UniqueIndex == 0 { idx.options |= collection.UniqueIndex } return idx } // CaseInsensitive returns a copy of the index that indexes record values ignoring casing. // Returned index contains previous options plus the case insensitivity one. func (idx Index) CaseInsensitive() Index { if idx.options&collection.CaseInsensitiveIndex == 0 { idx.options |= collection.CaseInsensitiveIndex } return idx }
10. #10query.gno
11. #11package datastore import ( "gno.land/p/moul/collection/v0" ) var defaultQuery = Query{indexName: collection.IDIndex} // Query contains arguments for querying a storage. type Query struct { offset int size int indexName string indexKey string } // Offset returns the position of the first record to return. // The minimum offset value is 0. func (q Query) Offset() int { return q.offset } // Size returns the maximum number of records a query returns. func (q Query) Size() int { return q.size } // IndexName returns the name of the storage index being used for the query. func (q Query) IndexName() string { return q.indexName } // IndexKey return the index key value to locate the records. // An empty string is returned when all indexed records match the query. func (q Query) IndexKey() string { return q.indexKey } // IsEmpty checks if the query is empty. // Empty queries return no records. func (q Query) IsEmpty() bool { return q.indexName == "" }
12. #12query_options.gno
13. #13package datastore import ( "errors" "strings" ) var ( ErrEmptyQueryIndexName = errors.New("query index name is empty") ErrInvalidQueryOffset = errors.New("minimum allowed query offset is 0") ErrInvalidQuerySize = errors.New("minimum allowed query size is 1") ) // QueryOption configures queries. type QueryOption func(*Query) error // WithOffset assigns the offset or position of the first record that query must return. // The minimum allowed offset is 0. func WithOffset(offset int) QueryOption { return func(q *Query) error { if offset < 0 { return ErrInvalidQueryOffset } q.offset = offset return nil } } // WithSize assigns the maximum number of records that query can return. // The minimum allowed size is 1. func WithSize(size int) QueryOption { return func(q *Query) error { if size < 1 { return ErrInvalidQuerySize } q.size = size return nil } } // UseIndex assigns the index that the query must use to get the records. // Using an index requires a key value to locate the records within the index. func UseIndex(name, key string) QueryOption { return func(q *Query) error { q.indexName = strings.TrimSpace(name) if q.indexName == "" { return ErrEmptyQueryIndexName } q.indexKey = key return nil } }
14. #14record.gno
15. #15package datastore import ( "errors" "gno.land/p/moul/collection/v0" "gno.land/p/nt/bptree/v0" "gno.land/p/nt/seqid/v0" ) // ErrUndefinedField indicates that a field in not defined in a record's schema. var ErrUndefinedField = errors.New("undefined field") type ( // Record stores values for one or more fields. Record interface { ReadOnlyRecord // Set assings a value to a record field. // If the field doesn't exist it's created if the underlying schema allows it. // Storage schema can optionally be strict in which case no new fields other than // the ones that were previously defined are allowed. Set(field string, value interface{}) error // Save assigns an ID to newly created records and update storage indexes. Save() bool } // ReadOnlyRecord defines an interface for read-only records. ReadOnlyRecord interface { // ID returns record's ID ID() uint64 // Key returns a string representation of the record's ID. // It's used to be able to search records within the ID index. Key() string // Type returns the record's type. Type() string // Fields returns the list of the record's field names. Fields() []string // IsEmpty checks if the record has no values. IsEmpty() bool // HasField checks if the record has a specific field. HasField(name string) bool // Get returns the value of a record's field. Get(field string) (value interface{}, found bool) // MustGet returns the value of a record's field or panics when the field is not found. MustGet(field string) interface{} } // RecordIterFn defines a type for record iteration functions. RecordIterFn func(Record) (stop bool) // Recordset defines an interface that allows iterating multiple records. Recordset interface { // Iterate iterates records in order. Iterate(fn RecordIterFn) (stopped bool) // ReverseIterate iterates records in reverse order. ReverseIterate(fn RecordIterFn) (stopped bool) // Size returns the number of records in the recordset. Size() int } ) type record struct { id uint64 schema *Schema collection *collection.Collection values *bptree.BPTree // string(field index) -> interface{} } // ID returns record's ID func (r record) ID() uint64 { return r.id } // Key returns a string representation of the record's ID. // It's used to be able to search records within the ID index. func (r record) Key() string { return seqid.ID(r.id).String() } // Type returns the record's type. func (r record) Type() string { return r.schema.Name() } // Fields returns the list of the record's field names. func (r record) Fields() []string { return r.schema.Fields() } // IsEmpty checks if the record has no values. func (r record) IsEmpty() bool { return r.values.Size() == 0 } // HasField checks if the record has a specific field. func (r record) HasField(name string) bool { return r.schema.HasField(name) } // Set assings a value to a record field. // If the field doesn't exist it's created if the underlying schema allows it. // Storage schema can optionally be strict in which case no new fields other than // the ones that were previously defined are allowed. func (r *record) Set(field string, value interface{}) error { i := r.schema.GetFieldIndex(field) if i == -1 { if r.schema.IsStrict() { return ErrUndefinedField } i, _ = r.schema.AddField(field, nil) } key := castIntToKey(i) r.values.Set(key, value) return nil } // Get returns the value of a record's field. func (r record) Get(field string) (value interface{}, found bool) { i := r.schema.GetFieldIndex(field) if i == -1 { return nil, false } key := castIntToKey(i) if r.values.Has(key) { return r.values.Get(key), true } return nil, false } // MustGet returns the value of a record's field or panics when the field is not found. func (r record) MustGet(field string) interface{} { v, found := r.Get(field) if !found { panic("field not found: " + field) } return v } // Save assigns an ID to newly created records and update storage indexes. func (r *record) Save() bool { if r.id == 0 { r.id = r.collection.Set(r) return r.id != 0 } return r.collection.Update(r.id, r) } type recordset struct { query Query records bptree.ITree keys []string size int } // Iterate iterates records in order. func (rs recordset) Iterate(fn RecordIterFn) (stopped bool) { if rs.isUsingCustomIndex() { for _, k := range rs.keys { r, ok := rs.records.Get(k).(Record) if !ok { continue } if fn(r) { return true } } return false } offset := rs.query.Offset() count := rs.query.Size() if count == 0 { count = rs.records.Size() } return rs.records.IterateByOffset(offset, count, func(_ string, v interface{}) bool { return fn(v.(Record)) }) } // ReverseIterate iterates records in reverse order. func (rs recordset) ReverseIterate(fn RecordIterFn) (stopped bool) { if rs.isUsingCustomIndex() { for i := len(rs.keys) - 1; i >= 0; i-- { r, ok := rs.records.Get(rs.keys[i]).(Record) if !ok { continue } if fn(r) { return true } } return false } offset := rs.query.Offset() count := rs.query.Size() if count == 0 { count = rs.records.Size() } return rs.records.ReverseIterateByOffset(offset, count, func(_ string, v interface{}) bool { return fn(v.(Record)) }) } // Size returns the number of records in the recordset. func (rs recordset) Size() int { return rs.size } func (rs recordset) isUsingCustomIndex() bool { return rs.query.IndexName() != collection.IDIndex }
16. #16schema.gno
17. #17package datastore import ( "encoding/binary" "gno.land/p/nt/bptree/list/v0" "gno.land/p/nt/bptree/v0" ) // TODO: Support versioning // Schema contains information about fields and default field values. // It also offers the possibility to configure it as static to indicate // that only configured fields should be allowed. type Schema struct { name string strict bool fields list.List // int(field index) -> string(field name) defaults *bptree.BPTree // string(field index) -> interface{} } // NewSchema creates a new schema. func NewSchema(name string, options ...SchemaOption) *Schema { s := &Schema{ name: name, defaults: bptree.NewBPTree32(), } for _, apply := range options { apply(s) } return s } // Name returns schema's name. func (s *Schema) Name() string { return s.name } // Fields returns the list field names that are defined in the schema. func (s *Schema) Fields() []string { fields := make([]string, s.fields.Len()) s.fields.ForEach(func(i int, v interface{}) bool { fields[i] = v.(string) return false }) return fields } // Size returns the number of fields the schema has. func (s *Schema) Size() int { return s.fields.Len() } // IsStrict check if the schema is configured as a strict one. func (s *Schema) IsStrict() bool { return s.strict } // HasField check is a field has been defined in the schema. func (s *Schema) HasField(name string) bool { return s.GetFieldIndex(name) >= 0 } // AddField adds a new field to the schema. // A default field value can be specified, otherwise `defaultValue` must be nil. func (s *Schema) AddField(name string, defaultValue interface{}) (index int, added bool) { if s.HasField(name) { return -1, false } s.fields.Append(name) index = s.fields.Len() - 1 if defaultValue != nil { key := castIntToKey(index) s.defaults.Set(key, defaultValue) } return index, true } // GetFieldIndex returns the index number of a schema field. // // Field index indicates the order the field has within the schema. // When defined fields are added they get an index starting from // field index 0. // // Fields are internally referenced by index number instead of the name // to be able to rename fields easily. func (s *Schema) GetFieldIndex(name string) int { index := -1 s.fields.ForEach(func(i int, v interface{}) bool { if name != v.(string) { return false } index = i return true }) return index } // GetFieldName returns the name of a field for a specific field index. func (s *Schema) GetFieldName(index int) (name string, found bool) { if v, ok := s.fields.Get(index); ok { return v.(string), true } return "", false } // GetDefault returns the default value for a field. func (s *Schema) GetDefault(name string) (value interface{}, found bool) { i := s.GetFieldIndex(name) if i == -1 { return nil, false } return s.GetDefaultByIndex(i) } // GetDefaultByIndex returns the default value for a field by it's index. func (s *Schema) GetDefaultByIndex(index int) (value interface{}, found bool) { key := castIntToKey(index) if !s.defaults.Has(key) { return nil, false } v := s.defaults.Get(key) if fn, ok := v.(func() interface{}); ok { return fn(), true } return v, true } // RenameField renames a field. func (s *Schema) RenameField(name, newName string) (renamed bool) { if s.HasField(newName) { return false } i := s.GetFieldIndex(name) if i == -1 { return false } s.fields.Set(i, newName) return true } func castIntToKey(i int) string { buf := make([]byte, 8) binary.BigEndian.PutUint64(buf, uint64(i)) return string(buf) }
18. #18schema_options.gno
19. #19package datastore import "strings" // StorageOption configures schemas. type SchemaOption func(*Schema) // WithField assign a new field to the schema definition. func WithField(name string) SchemaOption { return func(s *Schema) { name = strings.TrimSpace(name) if name != "" { s.fields.Append(name) } } } // WithDefaultField assign a new field with a default value to the schema definition. // Default value is assigned to newly created records asociated to to schema. func WithDefaultField(name string, value interface{}) SchemaOption { return func(s *Schema) { name = strings.TrimSpace(name) if name != "" { s.fields.Append(name) key := castIntToKey(s.fields.Len() - 1) s.defaults.Set(key, value) } } } // Strict configures the schema as a strict one. // By default schemas should allow the creation of any user defined field, // making them strict limits the allowed record fields to the ones pre-defined // in the schema. Fields are pre-defined using `WithField`, `WithDefaultField` // or by calling `Schema.AddField()`. func Strict() SchemaOption { return func(s *Schema) { s.strict = true } }
20. #20storage.gno
21. #21package datastore import ( "errors" "strings" "gno.land/p/moul/collection/v0" "gno.land/p/nt/bptree/v0" "gno.land/p/nt/seqid/v0" ) // NewStorage creates a new records storage. func NewStorage(name string, options ...StorageOption) Storage { s := Storage{ name: name, collection: collection.New(), schema: NewSchema(strings.Title(name)), } for _, apply := range options { apply(&s) } return s } // Storage stores a collection of records. // // By default it searches records by record ID but it allows // using custom user defined indexes for other record fields. // // When a storage is created it defines a default schema that // keeps track of record fields. Storage can be optionally // created with a user defined schema in cases where the number // of fields has to be pre-defined or when new records must have // one or more fields initialized to default values. type Storage struct { name string collection *collection.Collection schema *Schema } // Name returns storage's name. func (s Storage) Name() string { return s.name } // Collection returns the undelying collection used by the // storage to store all records. func (s Storage) Collection() *collection.Collection { return s.collection } // Schema returns the schema being used to track record fields. func (s Storage) Schema() *Schema { return s.schema } // Size returns the number of records that the storage have. func (s Storage) Size() int { return s.collection.GetIndex(collection.IDIndex).Size() } // NewRecord creates a new storage record. // // If a custom schema with default field values is assigned to // storage it's used to assign initial default values when new // records are created. // // Creating a new record doesn't assign an ID to it, a new ID // is generated and assigned to the record when it's saved for // the first time. func (s Storage) NewRecord() Record { r := &record{ schema: s.schema, collection: s.collection, values: bptree.NewBPTree32(), } // Assign default record values if the schema defines them for i, name := range s.schema.Fields() { if v, found := s.schema.GetDefaultByIndex(i); found { r.Set(name, v) } } return r } // Query returns a recordset that matches the query parameters. // By default query selects records using the ID index. // // Example usage: // // // Get 50 records starting from the one at position 100 // rs, _ := storage.Query( // WithOffset(100), // WithSize(50), // ) // // // Iterate records to create a new slice // var records []Record // rs.Iterate(func (r Record) bool { // records = append(records, r) // return false // }) func (s Storage) Query(options ...QueryOption) (Recordset, error) { // Initialize the recordset for the query rs := recordset{ query: defaultQuery, records: s.collection.GetIndex(collection.IDIndex), } for _, apply := range options { if err := apply(&rs.query); err != nil { return nil, err } } indexName := rs.query.IndexName() if indexName != collection.IDIndex { // When using a custom index get the keys to get records from the ID index keys, err := s.getIndexRecordsKeys(indexName, rs.query.IndexKey()) if err != nil { return nil, err } // Adjust the number of keys to match available query options if offset := rs.query.Offset(); offset > 0 { if offset > len(keys) { keys = nil } else { keys = keys[offset:] } } if size := rs.query.Size(); size > 0 && size < len(keys) { keys = keys[:size] } rs.keys = keys rs.size = len(keys) } else { // When using the default ID index init size with the total number of records rs.size = rs.records.Size() // Adjust recordset size to match available query options if offset := rs.query.Offset(); offset > 0 { if offset > rs.size { rs.size = 0 } else { rs.size -= offset } } if size := rs.query.Size(); size > 0 && size < rs.size { rs.size = size } } return rs, nil } // MustQuery returns a recordset that matches the query parameters or panics on error. // By default query selects records using the ID index. // // Example usage: // // // Get 50 records starting from the one at position 100 // var records []Record // storage.MustQuery( // WithOffset(100), // WithSize(50), // ).Iterate(func (r Record) bool { // records = append(records, r) // return false // }) func (s Storage) MustQuery(options ...QueryOption) Recordset { rs, err := s.Query(options...) if err != nil { panic(err) } return rs } // Get returns the first record found for a key within a storage index. // // This is a convenience method to get a single record. A multi index will // always return the first record value for the specified key in this case. // To get multiple records create a query using a custom index and key value // or use the underlying storage collection. func (s Storage) Get(indexName, indexKey string) (_ Record, found bool) { iter := s.collection.Get(indexName, indexKey) if iter.Next() { return iter.Value().Obj.(Record), true } return nil, false } // GetByID returns a record whose ID matches the specified ID. func (s Storage) GetByID(id uint64) (_ Record, found bool) { iter := s.collection.Get(collection.IDIndex, seqid.ID(id).String()) if iter.Next() { return iter.Value().Obj.(Record), true } return nil, false } // Delete deletes a record from the storage. func (s Storage) Delete(id uint64) bool { return s.collection.Delete(id) } func (s Storage) getIndexRecordsKeys(indexName, indexKey string) ([]string, error) { idx := s.collection.GetIndex(indexName) if idx == nil { return nil, errors.New("storage index for query not found: " + indexName) } keys := castIfaceToRecordKeys(idx.Get(indexKey)) if keys == nil { return nil, errors.New("unexpected storage index key format") } return keys, nil } func castIfaceToRecordKeys(v interface{}) []string { switch k := v.(type) { case []string: return k case string: return []string{k} } return nil }
22. #22storage_options.gno
23. #23package datastore // StorageOption configures storages. type StorageOption func(*Storage) // WithSchema assigns a schema to the storage. func WithSchema(s *Schema) StorageOption { return func(st *Storage) { if s != nil { st.schema = s } } } // WithIndex assigns an index to the storage. func WithIndex(i Index) StorageOption { return func(st *Storage) { st.collection.AddIndex(i.name, i.fn, i.options) } }
24. #24/gno.MemPackageType