Merge pull request 'refactor sqlite queue' (#10) from SpliteSqliteQueue into main

Reviewed-on: https://git.infra.nkode.tech/dkelly/go-nkode/pulls/10

cmd/main.go

@@ -1,14 +1,17 @@
 package main
 
 import (
+	"context"
+	"database/sql"
 	"fmt"
 	"github.com/google/uuid"
 	httpSwagger "github.com/swaggo/http-swagger"
 	_ "go-nkode/docs"
 	"go-nkode/internal/api"
-	"go-nkode/internal/db"
 	"go-nkode/internal/email"
 	"go-nkode/internal/models"
+	"go-nkode/internal/repository"
+	sqliteQueue "go-nkode/internal/sqlc"
 	"log"
 	"net/http"
 	"os"
@@ -37,24 +40,42 @@ const (
 // @securityDefinitions.apiKey  ApiKeyAuth
 // @in                          header
 // @name                        Authorization
-
 func main() {
 	dbPath := os.Getenv("SQLITE_DB")
 	if dbPath == "" {
-		log.Fatalf("SQLITE_DB=/path/to/nkode.db not set")
+		log.Fatal("SQLITE_DB=/path/to/nkode.db not set")
 	}
-	sqlitedb, err := db.NewSqliteDB(dbPath)
+
+	sqliteDb, err := sql.Open("sqlite3", dbPath)
 	if err != nil {
-		log.Fatalf("%v", err)
+		log.Fatalf("failed to open database: %v", err)
 	}
-	defer sqlitedb.Close()
+
+	if err := sqliteDb.Ping(); err != nil {
+		log.Fatalf("failed to connect to database: %v", err)
+	}
+
+	ctx := context.Background()
+	queue, err := sqliteQueue.NewQueue(sqliteDb, ctx)
+	if err != nil {
+		log.Fatal(err)
+	}
+	queue.Start()
+
+	defer func(queue *sqliteQueue.Queue) {
+		if err := queue.Stop(); err != nil {
+			log.Fatal(err)
+		}
+	}(queue)
 
 	sesClient := email.NewSESClient()
 	emailQueue := email.NewEmailQueue(emailQueueBufferSize, maxEmailsPerSecond, &sesClient)
 	emailQueue.Start()
 	defer emailQueue.Stop()
 
-	nkodeApi := api.NewNKodeAPI(sqlitedb, emailQueue)
+	sqlitedb := repository.NewSqliteRepository(queue, ctx)
+	nkodeApi := api.NewNKodeAPI(&sqlitedb, emailQueue)
+
 	AddDefaultCustomer(nkodeApi)
 	handler := api.NKodeHandler{Api: nkodeApi}
 

go.mod

@@ -1,8 +1,6 @@
 module go-nkode
 
-go 1.22.0
+go 1.23.0
-
-toolchain go1.23.0
 
 require (
 	github.com/aws/aws-sdk-go-v2 v1.31.0
@@ -12,7 +10,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	github.com/swaggo/http-swagger v1.3.4
 	github.com/swaggo/swag v1.16.4
 	github.com/swaggo/swag/example/celler v0.0.0-20241025062444-99698582709d
@@ -20,6 +18,7 @@ require (
 )
 
 require (
+	github.com/DonovanKelly/sugar-n-spice v1.0.1 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.35 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect

go.sum

@@ -1,3 +1,5 @@
+github.com/DonovanKelly/sugar-n-spice v1.0.1 h1:VsybiCHSziAqyPtbYF6GtkiJYYECWMHKN+EyEa6UVpA=
+github.com/DonovanKelly/sugar-n-spice v1.0.1/go.mod h1:/HQWoablLFCwsa4gwfzVBu80cI5A3dyO1uCiB11sup0=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
 github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
 github.com/aws/aws-sdk-go-v2 v1.31.0 h1:3V05LbxTSItI5kUqNwhJrrrY1BAXxXt0sN0l72QmG5U=
@@ -116,6 +118,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
 github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
 github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww=

internal/api/nkode_api.go

@@ -5,10 +5,10 @@ import (
 	"github.com/google/uuid"
 	"github.com/patrickmn/go-cache"
 	"go-nkode/config"
-	"go-nkode/internal/db"
 	"go-nkode/internal/email"
 	"go-nkode/internal/entities"
 	"go-nkode/internal/models"
+	"go-nkode/internal/repository"
 	"go-nkode/internal/security"
 	"log"
 	"os"
@@ -21,12 +21,12 @@ const (
 )
 
 type NKodeAPI struct {
-	Db                 db.CustomerUserRepository
+	Db                 repository.CustomerUserRepository
 	SignupSessionCache *cache.Cache
 	EmailQueue         *email.Queue
 }
 
-func NewNKodeAPI(db db.CustomerUserRepository, queue *email.Queue) NKodeAPI {
+func NewNKodeAPI(db repository.CustomerUserRepository, queue *email.Queue) NKodeAPI {
 	return NKodeAPI{
 		Db:                 db,
 		EmailQueue:         queue,

internal/api/nkode_api_test.go

@@ -1,12 +1,15 @@
 package api
 
 import (
+	"context"
 	"github.com/stretchr/testify/assert"
-	"go-nkode/internal/db"
 	"go-nkode/internal/email"
 	"go-nkode/internal/entities"
 	"go-nkode/internal/models"
+	"go-nkode/internal/repository"
 	"go-nkode/internal/security"
+	sqlite_queue "go-nkode/internal/sqlc"
+	"log"
 	"os"
 	"testing"
 )
@@ -15,22 +18,31 @@ func TestNKodeAPI(t *testing.T) {
 	//db1 := NewInMemoryDb()
 	//testNKodeAPI(t, &db1)
 
-	dbFile := os.Getenv("TEST_DB")
+	dbPath := os.Getenv("TEST_DB")
-
+	ctx := context.Background()
-	db2, err := db.NewSqliteDB(dbFile)
+	sqliteDb, err := sqlite_queue.OpenSqliteDb(dbPath)
 	assert.NoError(t, err)
-	defer db2.Close()
-	testNKodeAPI(t, db2)
 
-	//if _, err := os.Stat(dbFile); err == nil {
+	queue, err := sqlite_queue.NewQueue(sqliteDb, ctx)
-	//	err = os.Remove(dbFile)
+	assert.NoError(t, err)
+	queue.Start()
+	defer func(queue *sqlite_queue.Queue) {
+		if err := queue.Stop(); err != nil {
+			log.Fatal(err)
+		}
+	}(queue)
+	sqlitedb := repository.NewSqliteRepository(queue, ctx)
+	testNKodeAPI(t, &sqlitedb)
+
+	//if _, err := os.Stat(dbPath); err == nil {
+	//	err = os.Remove(dbPath)
 	//	assert.NoError(t, err)
 	//} else {
 	//	assert.NoError(t, err)
 	//}
 }
 
-func testNKodeAPI(t *testing.T, db db.CustomerUserRepository) {
+func testNKodeAPI(t *testing.T, db repository.CustomerUserRepository) {
 	bufferSize := 100
 	emailsPerSec := 14
 	testClient := email.TestEmailClient{}

internal/email/queue.go

@@ -163,6 +163,6 @@ func (q *Queue) Stop() {
 	q.stop = true
 	// Wait for all emails to be processed
 	q.wg.Wait()
-	// Close the email queue
+	// Stop the email queue
 	close(q.emailQueue)
 }

internal/email/queue_test.go

@@ -22,7 +22,7 @@ func TestEmailQueue(t *testing.T) {
 		}
 		queue.AddEmail(email)
 	}
-	// Close the queue after all emails are processed
+	// Stop the queue after all emails are processed
 	queue.Stop()
 
 	assert.Equal(t, queue.FailedSendCount, 0)

internal/entities/customer.go

@@ -1,6 +1,7 @@
 package entities
 
 import (
+	"github.com/DonovanKelly/sugar-n-spice/set"
 	"github.com/google/uuid"
 	"go-nkode/config"
 	"go-nkode/internal/models"
@@ -38,8 +39,8 @@ func (c *Customer) IsValidNKode(kp KeypadDimension, passcodeAttrIdx []int) error
 	if validIdx := kp.ValidateAttributeIndices(passcodeAttrIdx); !validIdx {
 		return config.ErrInvalidNKodeIdx
 	}
-	passcodeSetVals := make(utils.Set[uint64])
+	passcodeSetVals := make(set.Set[uint64])
-	passcodeAttrVals := make(utils.Set[uint64])
+	passcodeAttrVals := make(set.Set[uint64])
 	attrVals, err := c.Attributes.AttrValsForKp(kp)
 	if err != nil {
 		return err

internal/entities/keypad_dimension.go

@@ -1,8 +1,8 @@
 package entities
 
 import (
+	"github.com/DonovanKelly/sugar-n-spice/all"
 	"go-nkode/config"
-	py "go-nkode/internal/utils"
 )
 
 type KeypadDimension struct {
@@ -26,13 +26,13 @@ func (kp *KeypadDimension) IsValidKeypadDimension() error {
 }
 
 func (kp *KeypadDimension) ValidKeySelections(selectedKeys []int) bool {
-	return py.All[int](selectedKeys, func(idx int) bool {
+	return all.All[int](selectedKeys, func(idx int) bool {
 		return 0 <= idx && idx < kp.NumbOfKeys
 	})
 }
 
 func (kp *KeypadDimension) ValidateAttributeIndices(attrIndicies []int) bool {
-	return py.All[int](attrIndicies, func(i int) bool {
+	return all.All[int](attrIndicies, func(i int) bool {
 		return i >= 0 && i < kp.TotalAttrs()
 	})
 }

internal/entities/user_interface.go

@@ -1,10 +1,10 @@
 package entities
 
 import (
+	"github.com/DonovanKelly/sugar-n-spice/set"
 	"go-nkode/config"
 	"go-nkode/internal/models"
 	"go-nkode/internal/security"
-	"go-nkode/internal/utils"
 	"log"
 )
 
@@ -122,15 +122,15 @@ func (u *UserInterface) randomAttributeRotation() error {
 	return nil
 }
 
-func (u *UserInterface) AttributeAdjacencyGraph() (map[int]utils.Set[int], error) {
+func (u *UserInterface) AttributeAdjacencyGraph() (map[int]set.Set[int], error) {
 	interfaceKeypad, err := u.InterfaceMatrix()
 	if err != nil {
 		return nil, err
 	}
-	graph := make(map[int]utils.Set[int])
+	graph := make(map[int]set.Set[int])
 
 	for _, key := range interfaceKeypad {
-		keySet := utils.NewSetFromSlice(key)
+		keySet := set.NewSetFromSlice(key)
 		for _, attr := range key {
 			attrAdjacency := keySet.Copy()
 			attrAdjacency.Remove(attr)
@@ -154,7 +154,7 @@ func (u *UserInterface) LoginShuffle() error {
 	if err != nil {
 		return err
 	}
-	selectedSets := utils.NewSetFromSlice[int](setIdxs[:numbOfSelectedSets])
+	selectedSets := set.NewSetFromSlice[int](setIdxs[:numbOfSelectedSets])
 
 	for keyIdx, key := range keypadSet1 {
 		for idx := range key {

internal/entities/user_signup_session.go

@@ -1,11 +1,12 @@
 package entities
 
 import (
+	"github.com/DonovanKelly/sugar-n-spice/all"
+	"github.com/DonovanKelly/sugar-n-spice/set"
 	"github.com/google/uuid"
 	"go-nkode/config"
 	"go-nkode/internal/models"
 	"go-nkode/internal/security"
-	py "go-nkode/internal/utils"
 	"log"
 	"sort"
 )
@@ -50,7 +51,7 @@ func NewSignupResetSession(userEmail models.UserEmail, kp KeypadDimension, custo
 }
 
 func (s *UserSignSession) DeducePasscode(confirmKeyEntry models.KeySelection) ([]int, error) {
-	validEntry := py.All[int](confirmKeyEntry, func(i int) bool {
+	validEntry := all.All[int](confirmKeyEntry, func(i int) bool {
 		return 0 <= i && i < s.Kp.NumbOfKeys
 	})
 
@@ -93,8 +94,8 @@ func (s *UserSignSession) DeducePasscode(confirmKeyEntry models.KeySelection) ([
 	passcode := make([]int, passcodeLen)
 
 	for idx := 0; idx < passcodeLen; idx++ {
-		setKey := py.NewSetFromSlice[int](setKeyVals[idx])
+		setKey := set.NewSetFromSlice[int](setKeyVals[idx])
-		confirmKey := py.NewSetFromSlice[int](confirmKeyVals[idx])
+		confirmKey := set.NewSetFromSlice[int](confirmKeyVals[idx])
 		intersection := setKey.Intersect(confirmKey)
 		if intersection.Size() < 1 {
 			log.Printf("set and confirm do not intersect at index %d", idx)
@@ -111,7 +112,7 @@ func (s *UserSignSession) DeducePasscode(confirmKeyEntry models.KeySelection) ([
 }
 
 func (s *UserSignSession) SetUserNKode(keySelection models.KeySelection) (models.IdxInterface, error) {
-	validKeySelection := py.All[int](keySelection, func(i int) bool {
+	validKeySelection := all.All[int](keySelection, func(i int) bool {
 		return 0 <= i && i < s.Kp.NumbOfKeys
 	})
 	if !validKeySelection {

internal/entities/user_test.go

@@ -1,9 +1,9 @@
 package entities
 
 import (
+	"github.com/DonovanKelly/sugar-n-spice/all"
 	"github.com/stretchr/testify/assert"
 	"go-nkode/internal/models"
-	py "go-nkode/internal/utils"
 	"testing"
 )
 
@@ -120,12 +120,12 @@ func TestUserInterface_PartialInterfaceShuffle(t *testing.T) {
 		shuffleCompare[idx] = val == postShuffle[idx]
 	}
 
-	allTrue := py.All[bool](shuffleCompare, func(n bool) bool {
+	allTrue := all.All[bool](shuffleCompare, func(n bool) bool {
 		return n == true
 	})
 	assert.False(t, allTrue)
 
-	allFalse := py.All[bool](shuffleCompare, func(n bool) bool {
+	allFalse := all.All[bool](shuffleCompare, func(n bool) bool {
 		return n == false
 	})
 

internal/repository/customer_user_repository.go

@@ -1,4 +1,4 @@
-package db
+package repository
 
 import (
 	"go-nkode/internal/entities"

internal/repository/in_memory_db.go

@@ -1,4 +1,4 @@
-package db
+package repository
 
 import (
 	"errors"

internal/repository/sqlite_repository.go

@@ -1,4 +1,4 @@
-package db
+package repository
 
 import (
 	"context"
@@ -14,82 +14,21 @@ import (
 	"go-nkode/internal/sqlc"
 	"go-nkode/internal/utils"
 	"log"
-	"sync"
 )
 
-const writeBufferSize = 100
+type SqliteRepository struct {
-
+	Queue *sqlc.Queue
-type sqlcGeneric func(*sqlc.Queries, context.Context, any) error
-
-// WriteTx represents a write transaction
-type WriteTx struct {
-	ErrChan chan error
-	Query   sqlcGeneric
-	Args    interface{}
-}
-
-// SqliteDB represents the SQLite database connection and write queue
-type SqliteDB struct {
-	queries    *sqlc.Queries
-	db         *sql.DB
-	writeQueue chan WriteTx
-	wg         sync.WaitGroup
 	ctx   context.Context
-	cancel     context.CancelFunc
 }
 
-// NewSqliteDB initializes a new SqliteDB instance
+func NewSqliteRepository(queue *sqlc.Queue, ctx context.Context) SqliteRepository {
-func NewSqliteDB(path string) (*SqliteDB, error) {
+	return SqliteRepository{
-	if path == "" {
+		Queue: queue,
-		return nil, errors.New("database path is required")
-	}
-
-	db, err := sql.Open("sqlite3", path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open database: %w", err)
-	}
-
-	if err := db.Ping(); err != nil {
-		return nil, fmt.Errorf("failed to connect to database: %w", err)
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	sqldb := &SqliteDB{
-		queries:    sqlc.New(db),
-		db:         db,
-		writeQueue: make(chan WriteTx, writeBufferSize),
 		ctx:   ctx,
-		cancel:     cancel,
-	}
-
-	sqldb.wg.Add(1)
-	go sqldb.processWriteQueue()
-
-	return sqldb, nil
-}
-
-// processWriteQueue handles write transactions from the queue
-func (d *SqliteDB) processWriteQueue() {
-	defer d.wg.Done()
-	for {
-		select {
-		case <-d.ctx.Done():
-			return
-		case writeTx := <-d.writeQueue:
-			err := writeTx.Query(d.queries, d.ctx, writeTx.Args)
-			writeTx.ErrChan <- err
-		}
 	}
 }
 
-func (d *SqliteDB) Close() error {
+func (d *SqliteRepository) CreateCustomer(c entities.Customer) error {
-	d.cancel()
-	d.wg.Wait()
-	close(d.writeQueue)
-	return d.db.Close()
-}
-
-func (d *SqliteDB) CreateCustomer(c entities.Customer) error {
 	queryFunc := func(q *sqlc.Queries, ctx context.Context, args any) error {
 		params, ok := args.(sqlc.CreateCustomerParams)
 		if !ok {
@@ -98,10 +37,10 @@ func (d *SqliteDB) CreateCustomer(c entities.Customer) error {
 		return q.CreateCustomer(ctx, params)
 	}
 
-	return d.enqueueWriteTx(queryFunc, c.ToSqlcCreateCustomerParams())
+	return d.Queue.EnqueueWriteTx(queryFunc, c.ToSqlcCreateCustomerParams())
 }
 
-func (d *SqliteDB) WriteNewUser(u entities.User) error {
+func (d *SqliteRepository) WriteNewUser(u entities.User) error {
 	queryFunc := func(q *sqlc.Queries, ctx context.Context, args any) error {
 		params, ok := args.(sqlc.CreateUserParams)
 		if !ok {
@@ -109,7 +48,7 @@ func (d *SqliteDB) WriteNewUser(u entities.User) error {
 		}
 		return q.CreateUser(ctx, params)
 	}
-	// Use the wrapped function in enqueueWriteTx
+	// Use the wrapped function in EnqueueWriteTx
 
 	renew := 0
 	if u.Renew {
@@ -136,10 +75,10 @@ func (d *SqliteDB) WriteNewUser(u entities.User) error {
 		SvgIDInterface:   security.IntArrToByteArr(u.Interface.SvgId),
 		CreatedAt:        sql.NullString{String: utils.TimeStamp(), Valid: true},
 	}
-	return d.enqueueWriteTx(queryFunc, params)
+	return d.Queue.EnqueueWriteTx(queryFunc, params)
 }
 
-func (d *SqliteDB) UpdateUserNKode(u entities.User) error {
+func (d *SqliteRepository) UpdateUserNKode(u entities.User) error {
 	queryFunc := func(q *sqlc.Queries, ctx context.Context, args any) error {
 		params, ok := args.(sqlc.UpdateUserParams)
 		if !ok {
@@ -147,7 +86,7 @@ func (d *SqliteDB) UpdateUserNKode(u entities.User) error {
 		}
 		return q.UpdateUser(ctx, params)
 	}
-	// Use the wrapped function in enqueueWriteTx
+	// Use the wrapped function in EnqueueWriteTx
 	renew := 0
 	if u.Renew {
 		renew = 1
@@ -170,10 +109,10 @@ func (d *SqliteDB) UpdateUserNKode(u entities.User) error {
 		IdxInterface:     security.IntArrToByteArr(u.Interface.IdxInterface),
 		SvgIDInterface:   security.IntArrToByteArr(u.Interface.SvgId),
 	}
-	return d.enqueueWriteTx(queryFunc, params)
+	return d.Queue.EnqueueWriteTx(queryFunc, params)
 }
 
-func (d *SqliteDB) UpdateUserInterface(id models.UserId, ui entities.UserInterface) error {
+func (d *SqliteRepository) UpdateUserInterface(id models.UserId, ui entities.UserInterface) error {
 	queryFunc := func(q *sqlc.Queries, ctx context.Context, args any) error {
 		params, ok := args.(sqlc.UpdateUserInterfaceParams)
 		if !ok {
@@ -187,10 +126,10 @@ func (d *SqliteDB) UpdateUserInterface(id models.UserId, ui entities.UserInterfa
 		ID:           uuid.UUID(id).String(),
 	}
 
-	return d.enqueueWriteTx(queryFunc, params)
+	return d.Queue.EnqueueWriteTx(queryFunc, params)
 }
 
-func (d *SqliteDB) UpdateUserRefreshToken(id models.UserId, refreshToken string) error {
+func (d *SqliteRepository) UpdateUserRefreshToken(id models.UserId, refreshToken string) error {
 	queryFunc := func(q *sqlc.Queries, ctx context.Context, args any) error {
 		params, ok := args.(sqlc.UpdateUserRefreshTokenParams)
 		if !ok {
@@ -205,10 +144,10 @@ func (d *SqliteDB) UpdateUserRefreshToken(id models.UserId, refreshToken string)
 		},
 		ID: uuid.UUID(id).String(),
 	}
-	return d.enqueueWriteTx(queryFunc, params)
+	return d.Queue.EnqueueWriteTx(queryFunc, params)
 }
 
-func (d *SqliteDB) RenewCustomer(renewParams sqlc.RenewCustomerParams) error {
+func (d *SqliteRepository) RenewCustomer(renewParams sqlc.RenewCustomerParams) error {
 	queryFunc := func(q *sqlc.Queries, ctx context.Context, args any) error {
 		params, ok := args.(sqlc.RenewCustomerParams)
 		if !ok {
@@ -216,16 +155,16 @@ func (d *SqliteDB) RenewCustomer(renewParams sqlc.RenewCustomerParams) error {
 		}
 		return q.RenewCustomer(ctx, params)
 	}
-	return d.enqueueWriteTx(queryFunc, renewParams)
+	return d.Queue.EnqueueWriteTx(queryFunc, renewParams)
 }
 
-func (d *SqliteDB) Renew(id models.CustomerId) error {
+func (d *SqliteRepository) Renew(id models.CustomerId) error {
 	setXor, attrXor, err := d.renewCustomer(id)
 	if err != nil {
 		return err
 	}
 	customerId := models.CustomerIdToString(id)
-	userRenewRows, err := d.queries.GetUserRenew(d.ctx, customerId)
+	userRenewRows, err := d.Queue.Queries.GetUserRenew(d.ctx, customerId)
 	if err != nil {
 		return err
 	}
@@ -265,14 +204,14 @@ func (d *SqliteDB) Renew(id models.CustomerId) error {
 			Renew:    1,
 			ID:       uuid.UUID(user.Id).String(),
 		}
-		if err = d.enqueueWriteTx(queryFunc, params); err != nil {
+		if err = d.Queue.EnqueueWriteTx(queryFunc, params); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (d *SqliteDB) renewCustomer(id models.CustomerId) ([]uint64, []uint64, error) {
+func (d *SqliteRepository) renewCustomer(id models.CustomerId) ([]uint64, []uint64, error) {
 	customer, err := d.GetCustomer(id)
 	if err != nil {
 		return nil, nil, err
@@ -295,13 +234,13 @@ func (d *SqliteDB) renewCustomer(id models.CustomerId) ([]uint64, []uint64, erro
 		ID:              uuid.UUID(customer.Id).String(),
 	}
 
-	if err = d.enqueueWriteTx(queryFunc, params); err != nil {
+	if err = d.Queue.EnqueueWriteTx(queryFunc, params); err != nil {
 		return nil, nil, err
 	}
 	return setXor, attrXor, nil
 }
 
-func (d *SqliteDB) RefreshUserPasscode(user entities.User, passcodeIdx []int, customerAttr entities.CustomerAttributes) error {
+func (d *SqliteRepository) RefreshUserPasscode(user entities.User, passcodeIdx []int, customerAttr entities.CustomerAttributes) error {
 	if err := user.RefreshPasscode(passcodeIdx, customerAttr); err != nil {
 		return err
 	}
@@ -323,11 +262,11 @@ func (d *SqliteDB) RefreshUserPasscode(user entities.User, passcodeIdx []int, cu
 		Salt:     user.CipherKeys.Salt,
 		ID:       uuid.UUID(user.Id).String(),
 	}
-	return d.enqueueWriteTx(queryFunc, params)
+	return d.Queue.EnqueueWriteTx(queryFunc, params)
 }
 
-func (d *SqliteDB) GetCustomer(id models.CustomerId) (*entities.Customer, error) {
+func (d *SqliteRepository) GetCustomer(id models.CustomerId) (*entities.Customer, error) {
-	customer, err := d.queries.GetCustomer(d.ctx, uuid.UUID(id).String())
+	customer, err := d.Queue.Queries.GetCustomer(d.ctx, uuid.UUID(id).String())
 	if err != nil {
 		return nil, err
 	}
@@ -346,8 +285,8 @@ func (d *SqliteDB) GetCustomer(id models.CustomerId) (*entities.Customer, error)
 	}, nil
 }
 
-func (d *SqliteDB) GetUser(email models.UserEmail, customerId models.CustomerId) (*entities.User, error) {
+func (d *SqliteRepository) GetUser(email models.UserEmail, customerId models.CustomerId) (*entities.User, error) {
-	userRow, err := d.queries.GetUser(d.ctx, sqlc.GetUserParams{
+	userRow, err := d.Queue.Queries.GetUser(d.ctx, sqlc.GetUserParams{
 		Email:      string(email),
 		CustomerID: uuid.UUID(customerId).String(),
 	})
@@ -396,7 +335,7 @@ func (d *SqliteDB) GetUser(email models.UserEmail, customerId models.CustomerId)
 	return &user, nil
 }
 
-func (d *SqliteDB) RandomSvgInterface(kp entities.KeypadDimension) ([]string, error) {
+func (d *SqliteRepository) RandomSvgInterface(kp entities.KeypadDimension) ([]string, error) {
 	ids, err := d.getRandomIds(kp.TotalAttrs())
 	if err != nil {
 		return nil, err
@@ -404,18 +343,18 @@ func (d *SqliteDB) RandomSvgInterface(kp entities.KeypadDimension) ([]string, er
 	return d.getSvgsById(ids)
 }
 
-func (d *SqliteDB) RandomSvgIdxInterface(kp entities.KeypadDimension) (models.SvgIdInterface, error) {
+func (d *SqliteRepository) RandomSvgIdxInterface(kp entities.KeypadDimension) (models.SvgIdInterface, error) {
 	return d.getRandomIds(kp.TotalAttrs())
 }
 
-func (d *SqliteDB) GetSvgStringInterface(idxs models.SvgIdInterface) ([]string, error) {
+func (d *SqliteRepository) GetSvgStringInterface(idxs models.SvgIdInterface) ([]string, error) {
 	return d.getSvgsById(idxs)
 }
 
-func (d *SqliteDB) getSvgsById(ids []int) ([]string, error) {
+func (d *SqliteRepository) getSvgsById(ids []int) ([]string, error) {
 	svgs := make([]string, len(ids))
 	for idx, id := range ids {
-		svg, err := d.queries.GetSvgId(d.ctx, int64(id))
+		svg, err := d.Queue.Queries.GetSvgId(d.ctx, int64(id))
 		if err != nil {
 			return nil, err
 		}
@@ -424,25 +363,8 @@ func (d *SqliteDB) getSvgsById(ids []int) ([]string, error) {
 	return svgs, nil
 }
 
-func (d *SqliteDB) enqueueWriteTx(queryFunc sqlcGeneric, args any) error {
+func (d *SqliteRepository) getRandomIds(count int) ([]int, error) {
-	select {
+	tx, err := d.Queue.Db.Begin()
-	case <-d.ctx.Done():
-		return errors.New("database is shutting down")
-	default:
-	}
-
-	errChan := make(chan error, 1)
-	writeTx := WriteTx{
-		Query:   queryFunc,
-		Args:    args,
-		ErrChan: errChan,
-	}
-	d.writeQueue <- writeTx
-	return <-errChan
-}
-
-func (d *SqliteDB) getRandomIds(count int) ([]int, error) {
-	tx, err := d.db.Begin()
 	if err != nil {
 		log.Print(err)
 		return nil, config.ErrSqliteTx

internal/repository/sqlite_repository_test.go

@@ -1,28 +1,31 @@
-package db
+package repository
 
 import (
+	"context"
 	"github.com/stretchr/testify/assert"
 	"go-nkode/internal/entities"
 	"go-nkode/internal/models"
+	sqlite_queue "go-nkode/internal/sqlc"
 	"os"
 	"testing"
 )
 
 func TestNewSqliteDB(t *testing.T) {
-	dbFile := os.Getenv("TEST_DB")
+	dbPath := os.Getenv("TEST_DB")
 	//	sql_driver.MakeTables(dbFile)
-	db, err := NewSqliteDB(dbFile)
+	ctx := context.Background()
+	sqliteDb, err := sqlite_queue.OpenSqliteDb(dbPath)
 	assert.NoError(t, err)
-	defer db.Close()
 
-	testSignupLoginRenew(t, db)
+	queue, err := sqlite_queue.NewQueue(sqliteDb, ctx)
-	testSqliteDBRandomSvgInterface(t, db)
+	assert.NoError(t, err)
-	//	if _, err := os.Stat(dbFile); err == nil {
+
-	//		err = os.Remove(dbFile)
+	queue.Start()
-	//		assert.NoError(t, err)
+	defer queue.Stop()
-	//	} else {
+	db := NewSqliteRepository(queue, ctx)
-	//		assert.NoError(t, err)
+	assert.NoError(t, err)
-	//	}
+	testSignupLoginRenew(t, &db)
+	testSqliteDBRandomSvgInterface(t, &db)
 }
 
 func testSignupLoginRenew(t *testing.T, db CustomerUserRepository) {

internal/security/util.go

@@ -6,7 +6,7 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
-	"go-nkode/internal/utils"
+	"github.com/DonovanKelly/sugar-n-spice/set"
 	"log"
 	"math/big"
 	r "math/rand"
@@ -84,7 +84,7 @@ func GenerateRandomNonRepeatingUint64(listLen int) ([]uint64, error) {
 	if listLen > int(1)<<32 {
 		return nil, ErrRandNonRepeatingUint64
 	}
-	listSet := make(utils.Set[uint64])
+	listSet := make(set.Set[uint64])
 	for {
 		if listSet.Size() == listLen {
 			break
@@ -104,7 +104,7 @@ func GenerateRandomNonRepeatingInt(listLen int) ([]int, error) {
 	if listLen > int(1)<<31 {
 		return nil, ErrRandNonRepeatingInt
 	}
-	listSet := make(utils.Set[int])
+	listSet := make(set.Set[int])
 	for {
 		if listSet.Size() == listLen {
 			break

internal/sqlc/sqlite_queue.go

@@ -0,0 +1,93 @@
+package sqlc
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"sync"
+)
+
+const writeBufferSize = 100
+
+type SqlcGeneric func(*Queries, context.Context, any) error
+
+type WriteTx struct {
+	ErrChan chan error
+	Query   SqlcGeneric
+	Args    interface{}
+}
+
+type Queue struct {
+	Queries    *Queries
+	Db         *sql.DB
+	WriteQueue chan WriteTx
+	wg         sync.WaitGroup
+	ctx        context.Context
+	cancel     context.CancelFunc
+}
+
+func NewQueue(sqlDb *sql.DB, ctx context.Context) (*Queue, error) {
+	ctx, cancel := context.WithCancel(context.Background())
+	sqldb := &Queue{
+		Queries:    New(sqlDb),
+		Db:         sqlDb,
+		WriteQueue: make(chan WriteTx, writeBufferSize),
+		ctx:        ctx,
+		cancel:     cancel,
+	}
+
+	return sqldb, nil
+}
+
+func (d *Queue) Start() {
+	d.wg.Add(1)
+	defer d.wg.Done()
+	go func() {
+		for {
+			select {
+			case <-d.ctx.Done():
+				return
+			case writeTx := <-d.WriteQueue:
+				err := writeTx.Query(d.Queries, d.ctx, writeTx.Args)
+				writeTx.ErrChan <- err
+			}
+		}
+	}()
+}
+
+func (d *Queue) Stop() error {
+	d.cancel()
+	d.wg.Wait()
+	close(d.WriteQueue)
+	return d.Db.Close()
+}
+
+func (d *Queue) EnqueueWriteTx(queryFunc SqlcGeneric, args any) error {
+	select {
+	case <-d.ctx.Done():
+		return errors.New("database is shutting down")
+	default:
+	}
+
+	errChan := make(chan error, 1)
+	writeTx := WriteTx{
+		Query:   queryFunc,
+		Args:    args,
+		ErrChan: errChan,
+	}
+	d.WriteQueue <- writeTx
+	return <-errChan
+}
+
+func OpenSqliteDb(dbPath string) (*sql.DB, error) {
+	sqliteDb, err := sql.Open("sqlite3", dbPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open database: %w", err)
+	}
+
+	if err := sqliteDb.Ping(); err != nil {
+		return nil, fmt.Errorf("failed to connect to database: %w", err)
+	}
+	return sqliteDb, nil
+}

internal/utils/hashset.go

@@ -1,63 +0,0 @@
-package utils
-
-type Set[T comparable] map[T]struct{}
-
-func (s *Set[T]) Add(element T) {
-	(*s)[element] = struct{}{}
-}
-
-func (s *Set[T]) Remove(element T) {
-	delete(*s, element)
-}
-
-func (s *Set[T]) Contains(element T) bool {
-	_, exists := (*s)[element]
-	return exists
-}
-
-func (s *Set[T]) Size() int {
-	return len(*s)
-}
-
-func (s *Set[T]) ToSlice() []T {
-	list := make([]T, 0, len(*s))
-	for key := range *s {
-		list = append(list, key)
-	}
-	return list
-}
-
-func NewSetFromSlice[T comparable](slice []T) Set[T] {
-	set := make(Set[T])
-	for _, val := range slice {
-		set.Add(val)
-	}
-	return set
-}
-
-func (s *Set[T]) Copy() Set[T] {
-	newSet := make(Set[T])
-	for key, val := range *s {
-		newSet[key] = val
-	}
-	return newSet
-}
-
-func (s *Set[T]) IsDisjoint(otherSet Set[T]) bool {
-	for attr := range *s {
-		if otherSet.Contains(attr) {
-			return false
-		}
-	}
-	return true
-}
-
-func (s *Set[T]) Intersect(otherSet Set[T]) Set[T] {
-	intersect := make(Set[T])
-	for val := range *s {
-		if otherSet.Contains(val) {
-			intersect.Add(val)
-		}
-	}
-	return intersect
-}

internal/utils/hashset_test.go

@@ -1,35 +0,0 @@
-package utils
-
-import (
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-func TestSet(t *testing.T) {
-	intSet := make(Set[int])
-	intSet.Add(1)
-	intSet.Add(2)
-	assert.EqualValues(t, intSet.Size(), 2)
-	intSet.Add(3)
-	intSet.Add(3)
-	assert.EqualValues(t, intSet.Size(), 3)
-	intSet.Remove(2)
-	assert.EqualValues(t, intSet.Size(), 2)
-	assert.False(t, intSet.Contains(2))
-	assert.True(t, intSet.Contains(1))
-
-	list := intSet.ToSlice()
-	assert.Contains(t, list, 1)
-	assert.Contains(t, list, 3)
-}
-
-func TestSet_Copy(t *testing.T) {
-	intSet := NewSetFromSlice[int]([]int{1, 2, 3})
-
-	copySet := intSet.Copy()
-
-	intSet.Remove(1)
-	assert.Equal(t, intSet.Size(), 2)
-	assert.Equal(t, copySet.Size(), 3)
-
-}

internal/utils/py-builtin.go

@@ -1,11 +0,0 @@
-package utils
-
-func All[T comparable](slice []T, condition func(T) bool) bool {
-
-	for _, v := range slice {
-		if !condition(v) {
-			return false
-		}
-	}
-	return true
-}