Testing with leveldb

gosl.go

@@ -13,6 +13,8 @@ import (
 	"github.com/fsnotify/fsnotify"
 	"github.com/op/go-logging"
 	"github.com/spf13/viper"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
 	"github.com/tidwall/buntdb"
 	"gopkg.in/natefinch/lumberjack.v2"
 	"io"
@@ -82,7 +84,7 @@ func loadConfiguration() {
 	*goslConfig.isServer = viper.GetBool("config.isServer")
 	viper.SetDefault("config.isShell", false)
 	*goslConfig.isShell = viper.GetBool("config.isShell")
-	viper.SetDefault("config.database", "badger")
+	viper.SetDefault("config.database", "badger") // currently, badger, boltdb, leveldb
 	*goslConfig.database = viper.GetString("config.database")
 	viper.SetDefault("config.importFilename", "") // must be empty by default
 	*goslConfig.importFilename = viper.GetString("config.importFilename")
@@ -220,7 +222,7 @@ func main() {
 		checkErrPanic(err)
 		err = txn.Commit(nil)
 		checkErrPanic(err)
-		log.Debugf("SET %+v (json: %v)\n", testValue, string(jsonTestValue))
+		log.Debugf("badger SET %+v (json: %v)\n", testValue, string(jsonTestValue))
 		kv.Close()
 	} else if *goslConfig.database == "buntdb" {
 		/* NOTE(gwyneth): this fails because pointers to strings do not implement len(). Duh! 
@@ -235,10 +237,18 @@ func main() {
 			return err
 		})
 		checkErr(err)
-		log.Debugf("SET %+v (json: %v)\n", testValue, string(jsonTestValue))
+		log.Debugf("buntdb SET %+v (json: %v)\n", testValue, string(jsonTestValue))
+		db.Close()
+	} else if *goslConfig.database == "leveldb" {
+		goslConfig.dbNamePath = *goslConfig.myDir + string(os.PathSeparator) + databaseName
+		db, err := leveldb.OpenFile(goslConfig.dbNamePath, nil)
+		checkErrPanic(err)
+		err = db.Put([]byte(testAvatarName), jsonTestValue, nil)
+		checkErrPanic(err)
+		log.Debugf("leveldb SET %+v (json: %v)\n", testValue, string(jsonTestValue))
 		db.Close()
 	}
-	// common to both databases:
+	// common to all databases:
 	key, grid := searchKVname(testAvatarName)
 	log.Debugf("GET '%s' returned '%s' [grid '%s']\n", testAvatarName, key, grid)
 	log.Info("KV database seems fine.")
@@ -350,6 +360,12 @@ func handler(w http.ResponseWriter, r *http.Request) {
 					return err
 				})
 				checkErr(err)
+			} else if *goslConfig.database == "leveldb" {
+				db, err := leveldb.OpenFile(goslConfig.dbNamePath, nil)
+				checkErrPanic(err)
+				err = db.Put([]byte(name), jsonValueToInsert, nil)
+				checkErrPanic(err)
+				db.Close()				
 			}
 			messageToSL += "Added new entry for '" + name + "' which is: " + valueToInsert.UUID + " from grid: '" + valueToInsert.Grid + "'"
 		} else {
@@ -417,10 +433,21 @@ func searchKVname(avatarName string) (UUID string, grid string) {
 		if err != nil {
 			log.Errorf("Error while unparsing UUID for name: '%s' (%v)\n", avatarName, err)
     	}			
+	} else if *goslConfig.database == "leveldb" {
+		db, err := leveldb.OpenFile(goslConfig.dbNamePath, nil)
+		checkErrPanic(err)
+		defer db.Close()		
+		data, err := db.Get([]byte(avatarName), nil)
+		if err != nil {
+			log.Errorf("Error while getting UUID for name: '%s' (%v)\n", avatarName, err)
+    	} else {
+	    	err = json.Unmarshal(data, &val)
+			if err != nil {
+				log.Errorf("Error while unparsing UUID for name: '%s' (%v)\n", avatarName, err)
+    		}
+    	}			
 	}
-	time_end := time.Now()
-	diffTime := time_end.Sub(time_start)
-	log.Debugf("Time to lookup '%s': %v\n", avatarName, diffTime)
+	log.Debugf("Time to lookup '%s': %v\n", avatarName, time.Since(time_start))
 	if err != nil {
 		return NullUUID, ""
 	} // else:
@@ -472,7 +499,7 @@ func searchKVUUID(avatarKey string) (name string, grid string) {
 			return nil
 		})
 		kv.Close()
-	} else {
+	} else if *goslConfig.database == "buntdb" {
 		db, err := buntdb.Open(goslConfig.dbNamePath)
 		checkErrPanic(err)
 		err = db.View(func(tx *buntdb.Tx) error {
@@ -491,10 +518,31 @@ func searchKVUUID(avatarKey string) (name string, grid string) {
 		    return err
 		})
 		db.Close()	
+	} else if *goslConfig.database == "leveldb" {
+		db, err := leveldb.OpenFile(goslConfig.dbNamePath, nil)
+		checkErrPanic(err)
+		iter := db.NewIterator(nil, nil)
+		for iter.Next() {
+			// Remember that the contents of the returned slice should not be modified, and
+			// only valid until the next call to Next.
+			key := iter.Key()
+			value := iter.Value()
+	    	err = json.Unmarshal(value, &val)
+			if err != nil {
+				log.Errorf("Error '%s' while unparsing UUID for data: %v\n", err, value)
+				continue // a bit insane, but at least we will skip a few broken records 
+	    	}
+			checks++	//Just to see how many checks we made, for statistical purposes
+			if avatarKey == val.UUID {
+				found = string(key)
+				break
+			}
+		}
+		iter.Release()
+		err = iter.Error()
+		db.Close()
 	}
-	time_end := time.Now()
-	diffTime := time_end.Sub(time_start)
-	log.Debugf("Made %d checks for '%s' in %v\n", checks, avatarKey, diffTime)
+	log.Debugf("Made %d checks for '%s' in %v\n", checks, avatarKey, time.Since(time_start))
 	return found, val.Grid
 }
 
@@ -554,7 +602,7 @@ func importDatabase(filename string) {
 		    log.Fatal(err)
 		}
 		kv.PurgeOlderVersions()
-	} else {
+	} else if *goslConfig.database == "buntdb" {
 		db, err := buntdb.Open(goslConfig.dbNamePath)
 		checkErrPanic(err)
 		defer db.Close()
@@ -571,7 +619,7 @@ func importDatabase(filename string) {
 			} else if err != nil {
 				log.Fatal(err)
 			}
-			jsonNewEntry, err := json.Marshal(avatarUUID{ record[0], "Production" }) // W-Hat keys come all from the main LL grid, known as 'Production'
+			jsonNewEntry, err := json.Marshal(avatarUUID{ record[0], "Production" })
 			if err != nil {
 				log.Warning(err)
 			} else {			 
@@ -598,10 +646,45 @@ func importDatabase(filename string) {
 		    log.Fatal(err)
 		}			
 		db.Shrink()
+	} else if *goslConfig.database == "leveldb" {
+		db, err := leveldb.OpenFile(goslConfig.dbNamePath, nil)
+		checkErrPanic(err)
+		defer db.Close()
+		batch := new(leveldb.Batch)
+
+		for ;;limit++ {
+			record, err := cr.Read()
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				log.Fatal(err)
+			}
+			jsonNewEntry, err := json.Marshal(avatarUUID{ record[0], "Production" })
+			if err != nil {
+				log.Warning(err)
+			} else {
+				batch.Put([]byte(record[1]), jsonNewEntry)
+			}
+			if limit % goslConfig.BATCH_BLOCK == 0 && limit != 0 {
+				log.Info("Processing:", limit)		
+				err = db.Write(batch, nil)
+				if err != nil {
+				    log.Fatal(err)
+				}
+				batch.Reset()	// unlike the others, we don't need to create a new batch every time
+				runtime.GC()	// it never hurts...
+			}
+		}
+		// commit last batch
+		err = db.Write(batch, nil)
+		if err != nil {
+		    log.Fatal(err)
+		}
+		batch.Reset()	// reset it and let the garbage collector run
+		runtime.GC()
+		db.CompactRange(util.Range{ nil, nil })	
 	}
-	time_end := time.Now()
-	diffTime := time_end.Sub(time_start)
-	log.Info("Total read", limit, "records (or thereabouts) in", diffTime)
+	log.Info("Total read", limit, "records (or thereabouts) in", time.Since(time_start))
 }
 
 // NOTE(gwyneth): Auxiliary functions which I'm always using...