1
0

gosl.go 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745
  1. // gosl is a basic example of how to develop external web services for Second Life/OpenSimulator using the Go programming language.
  2. package main
  3. import (
  4. "bufio"
  5. "compress/bzip2"
  6. "encoding/csv"
  7. "encoding/json"
  8. "flag"
  9. "fmt"
  10. "github.com/dgraph-io/badger"
  11. "github.com/dgraph-io/badger/options"
  12. "github.com/fsnotify/fsnotify"
  13. "github.com/op/go-logging"
  14. "github.com/spf13/viper"
  15. "github.com/syndtr/goleveldb/leveldb"
  16. "github.com/syndtr/goleveldb/leveldb/util"
  17. "github.com/tidwall/buntdb"
  18. "gopkg.in/natefinch/lumberjack.v2"
  19. "io"
  20. "net/http"
  21. "net/http/fcgi"
  22. "os"
  23. "path/filepath"
  24. "regexp"
  25. "runtime"
  26. "strings"
  27. "time"
  28. )
  29. const NullUUID = "00000000-0000-0000-0000-000000000000" // always useful when we deal with SL/OpenSimulator...
  30. const databaseName = "gosl-database.db" // for BuntDB
  31. // Logging setup.
  32. var log = logging.MustGetLogger("gosl") // configuration for the go-logging logger, must be available everywhere
  33. var logFormat logging.Formatter
  34. // Opt is used for Badger database setup.
  35. var Opt badger.Options
  36. // AvatarUUID is the type that we store in the database; we keep a record from which grid it came from.
  37. type avatarUUID struct {
  38. UUID string // needs to be capitalised for JSON marshalling (it has to do with the way it works)
  39. Grid string
  40. }
  41. /*
  42. .__
  43. _____ _____ |__| ____
  44. / \\__ \ | |/ \
  45. | Y Y \/ __ \| | | \
  46. |__|_| (____ /__|___| /
  47. \/ \/ \/
  48. */
  49. // Configuration options
  50. type goslConfigOptions struct {
  51. BATCH_BLOCK int // how many entries to write to the database as a block; the bigger, the faster, but the more memory it consumes
  52. noMemory, isServer, isShell *bool
  53. myDir, myPort, importFilename, database *string
  54. dbNamePath string // for BuntDB
  55. logFilename string // for logs
  56. maxSize, maxBackups, maxAge int // logs configuration option
  57. }
  58. var goslConfig goslConfigOptions
  59. // loadConfiguration reads our configuration from a config.toml file
  60. func loadConfiguration() {
  61. fmt.Print("Reading gosl-basic configuration:") // note that we might not have go-logging active as yet, so we use fmt
  62. // Open our config file and extract relevant data from there
  63. err := viper.ReadInConfig() // Find and read the config file
  64. if err != nil {
  65. fmt.Println("Error reading config file:", err)
  66. return // we might still get away with this!
  67. }
  68. viper.SetDefault("config.BATCH_BLOCK", 100000) // NOTE(gwyneth): the authors of say that 100000 is way too much for Badger // NOTE(gwyneth): let's see what happens with BuntDB
  69. goslConfig.BATCH_BLOCK = viper.GetInt("config.BATCH_BLOCK")
  70. viper.SetDefault("config.myPort", 3000)
  71. *goslConfig.myPort = viper.GetString("config.myPort")
  72. viper.SetDefault("config.myDir", "slkvdb")
  73. *goslConfig.myDir = viper.GetString("config.myDir")
  74. viper.SetDefault("config.isServer", false)
  75. *goslConfig.isServer = viper.GetBool("config.isServer")
  76. viper.SetDefault("config.isShell", false)
  77. *goslConfig.isShell = viper.GetBool("config.isShell")
  78. viper.SetDefault("config.database", "badger") // currently, badger, boltdb, leveldb
  79. *goslConfig.database = viper.GetString("config.database")
  80. viper.SetDefault("config.importFilename", "") // must be empty by default
  81. *goslConfig.importFilename = viper.GetString("config.importFilename")
  82. viper.SetDefault("config.noMemory", false)
  83. *goslConfig.noMemory = viper.GetBool("config.noMemory")
  84. // Logging options
  85. viper.SetDefault("config.logFilename", "gosl.log")
  86. goslConfig.logFilename = viper.GetString("config.logFilename")
  87. viper.SetDefault("config.maxSize", 10)
  88. goslConfig.maxSize = viper.GetInt("config.maxSize")
  89. viper.SetDefault("config.maxBackups", 3)
  90. goslConfig.maxBackups = viper.GetInt("config.maxBackups")
  91. viper.SetDefault("config.maxAge", 28)
  92. goslConfig.maxAge = viper.GetInt("config.maxAge")
  93. }
  94. // main() starts here.
  95. func main() {
  96. // Flag setup; can be overridden by config file (I need to fix this to be the oher way round).
  97. goslConfig.myPort = flag.String("port", "3000", "Server port")
  98. goslConfig.myDir = flag.String("dir", "slkvdb", "Directory where database files are stored")
  99. goslConfig.isServer = flag.Bool("server", false, "Run as server on port " + *goslConfig.myPort)
  100. goslConfig.isShell = flag.Bool("shell", false, "Run as an interactive shell")
  101. goslConfig.importFilename = flag.String("import", "name2key.csv.bz2", "Import database from W-Hat (use the csv.bz2 version)")
  102. goslConfig.database = flag.String("database", "badger", "Database type (badger, buntdb, leveldb)")
  103. goslConfig.noMemory = flag.Bool("nomemory", false, "Attempt to use only disk to save memory on Badger (important for shared webservers)")
  104. // Config viper, which reads in the configuration file every time it's needed.
  105. // Note that we need some hard-coded variables for the path and config file name.
  106. viper.SetConfigName("config")
  107. viper.SetConfigType("toml") // just to make sure; it's the same format as OpenSimulator (or MySQL) config files
  108. viper.AddConfigPath("$HOME/go/src/gosl-basics/") // that's how I have it
  109. viper.AddConfigPath("$HOME/go/src/git.gwynethllewelyn.net/GwynethLlewelyn/gosl-basics/") // that's how you'll have it
  110. viper.AddConfigPath(".") // optionally look for config in the working directory
  111. loadConfiguration()
  112. // this will allow our configuration file to be 'read on demand'
  113. viper.WatchConfig()
  114. viper.OnConfigChange(func(e fsnotify.Event) {
  115. if *goslConfig.isServer || *goslConfig.isShell {
  116. fmt.Println("Config file changed:", e.Name) // BUG(gwyneth): FastCGI cannot write to output
  117. }
  118. loadConfiguration()
  119. })
  120. // default is FastCGI
  121. flag.Parse()
  122. // NOTE(gwyneth): We cannot write to stdout if we're running as FastCGI, only to logs!
  123. if *goslConfig.isServer || *goslConfig.isShell {
  124. fmt.Println("gosl is starting...")
  125. }
  126. // Setup the lumberjack rotating logger. This is because we need it for the go-logging logger when writing to files. (20170813)
  127. rotatingLogger := &lumberjack.Logger{
  128. Filename: goslConfig.logFilename,
  129. MaxSize: goslConfig.maxSize, // megabytes
  130. MaxBackups: goslConfig.maxBackups,
  131. MaxAge: goslConfig.maxAge, //days
  132. }
  133. // Set formatting for stderr and file (basically the same).
  134. logFormat := logging.MustStringFormatter(`%{color}%{time:2006/01/02 15:04:05.0} %{shortfile} - %{shortfunc} ▶ %{level:.4s}%{color:reset} %{message}`) // must be initialised or all hell breaks loose
  135. // Setup the go-logging Logger. Do **not** log to stderr if running as FastCGI!
  136. backendFile := logging.NewLogBackend(rotatingLogger, "", 0)
  137. backendFileFormatter := logging.NewBackendFormatter(backendFile, logFormat)
  138. backendFileLeveled := logging.AddModuleLevel(backendFileFormatter)
  139. backendFileLeveled.SetLevel(logging.INFO, "gosl") // we just send debug data to logs if we run as shell
  140. if *goslConfig.isServer || *goslConfig.isShell {
  141. backendStderr := logging.NewLogBackend(os.Stderr, "", 0)
  142. backendStderrFormatter := logging.NewBackendFormatter(backendStderr, logFormat)
  143. backendStderrLeveled := logging.AddModuleLevel(backendStderrFormatter)
  144. if *goslConfig.isShell {
  145. backendStderrLeveled.SetLevel(logging.DEBUG, "gosl") // shell is meant to be for debugging mostly
  146. } else {
  147. backendStderrLeveled.SetLevel(logging.INFO, "gosl")
  148. }
  149. logging.SetBackend(backendStderrLeveled, backendFileLeveled)
  150. } else {
  151. logging.SetBackend(backendFileLeveled) // FastCGI only logs to file
  152. }
  153. // Check if this directory actually exists; if not, create it. Panic if something wrong happens (we cannot proceed without a valid directory for the database to be written
  154. if stat, err := os.Stat(*goslConfig.myDir); err == nil && stat.IsDir() {
  155. // path is a valid directory
  156. log.Infof("Valid directory: %s\n", *goslConfig.myDir)
  157. } else {
  158. // try to create directory
  159. err = os.Mkdir(*goslConfig.myDir, 0700)
  160. checkErrPanic(err) // cannot make directory, panic and exit logging what went wrong
  161. log.Debugf("Created new directory: %s\n", *goslConfig.myDir)
  162. }
  163. if *goslConfig.database == "badger" {
  164. Opt = badger.DefaultOptions
  165. Opt.Dir = *goslConfig.myDir
  166. Opt.ValueDir = Opt.Dir
  167. Opt.TableLoadingMode = options.MemoryMap
  168. //Opt.TableLoadingMode = options.FileIO
  169. if *goslConfig.noMemory {
  170. // Opt.TableLoadingMode = options.FileIO // use standard file I/O operations for tables instead of LoadRAM
  171. Opt.TableLoadingMode = options.MemoryMap // MemoryMap indicates that that the file must be memory-mapped - https://github.com/dgraph-io/badger/issues/224#issuecomment-329643771
  172. // Opt.TableLoadingMode = options.FileIO
  173. // Opt.ValueLogFileSize = 1048576
  174. Opt.MaxTableSize = 1048576 // * 12
  175. Opt.LevelSizeMultiplier = 1
  176. Opt.NumMemtables = 1
  177. // Opt.MaxLevels = 10
  178. // Opt.SyncWrites = false
  179. // Opt.NumCompactors = 10
  180. // Opt.NumLevelZeroTables = 10
  181. // Opt.maxBatchSize =
  182. // Opt.maxBatchCount =
  183. goslConfig.BATCH_BLOCK = 1000 // try to import less at each time, it will take longer but hopefully work
  184. log.Info("Trying to avoid too much memory consumption")
  185. }
  186. }
  187. // Do some testing to see if the database is available
  188. const testAvatarName = "Nobody Here"
  189. var err error
  190. log.Info("gosl started and logging is set up. Proceeding to test database (" + *goslConfig.database + ") at " + *goslConfig.myDir)
  191. var testValue = avatarUUID{ NullUUID, "all grids" }
  192. jsonTestValue, err := json.Marshal(testValue)
  193. checkErrPanic(err) // something went VERY wrong
  194. if *goslConfig.database == "badger" {
  195. kv, err := badger.Open(Opt)
  196. checkErrPanic(err) // should probably panic, cannot prep new database
  197. txn := kv.NewTransaction(true)
  198. err = txn.Set([]byte(testAvatarName), jsonTestValue)
  199. checkErrPanic(err)
  200. err = txn.Commit(nil)
  201. checkErrPanic(err)
  202. log.Debugf("badger SET %+v (json: %v)\n", testValue, string(jsonTestValue))
  203. kv.Close()
  204. } else if *goslConfig.database == "buntdb" {
  205. /* NOTE(gwyneth): this fails because pointers to strings do not implement len(). Duh!
  206. if *goslConfig.myDir[len(*goslConfig.myDir)-1] != os.PathSeparator {
  207. *goslConfig.myDir = append(*goslConfig.myDir + os.PathSeparator
  208. } */
  209. goslConfig.dbNamePath = *goslConfig.myDir + string(os.PathSeparator) + databaseName
  210. db, err := buntdb.Open(goslConfig.dbNamePath)
  211. checkErrPanic(err)
  212. err = db.Update(func(tx *buntdb.Tx) error {
  213. _, _, err := tx.Set(testAvatarName, string(jsonTestValue), nil)
  214. return err
  215. })
  216. checkErr(err)
  217. log.Debugf("buntdb SET %+v (json: %v)\n", testValue, string(jsonTestValue))
  218. db.Close()
  219. } else if *goslConfig.database == "leveldb" {
  220. goslConfig.dbNamePath = *goslConfig.myDir + string(os.PathSeparator) + databaseName
  221. db, err := leveldb.OpenFile(goslConfig.dbNamePath, nil)
  222. checkErrPanic(err)
  223. err = db.Put([]byte(testAvatarName), jsonTestValue, nil)
  224. checkErrPanic(err)
  225. log.Debugf("leveldb SET %+v (json: %v)\n", testValue, string(jsonTestValue))
  226. db.Close()
  227. }
  228. // common to all databases:
  229. key, grid := searchKVname(testAvatarName)
  230. log.Debugf("GET '%s' returned '%s' [grid '%s']\n", testAvatarName, key, grid)
  231. log.Info("KV database seems fine.")
  232. if *goslConfig.importFilename != "" {
  233. log.Info("Attempting to import", *goslConfig.importFilename, "...")
  234. importDatabase(*goslConfig.importFilename)
  235. log.Info("Database finished import.")
  236. }
  237. if *goslConfig.isShell {
  238. log.Info("Starting to run as interactive shell")
  239. reader := bufio.NewReader(os.Stdin)
  240. fmt.Println("Ctrl-C to quit.")
  241. var err error // to avoid assigning text in a different scope (this is a bit awkward, but that's the problem with bi-assignment)
  242. var checkInput, avatarName, avatarKey, gridName string
  243. for {
  244. // Prompt and read
  245. fmt.Print("Enter avatar name or UUID: ")
  246. checkInput, err = reader.ReadString('\n')
  247. checkErr(err)
  248. checkInput = strings.TrimRight(checkInput, "\r\n")
  249. // fmt.Printf("Ok, got %s length is %d and UUID is %v\n", checkInput, len(checkInput), isValidUUID(checkInput))
  250. if (len(checkInput) == 36) && isValidUUID(checkInput) {
  251. avatarName, gridName = searchKVUUID(checkInput)
  252. avatarKey = checkInput
  253. } else {
  254. avatarKey, gridName = searchKVname(checkInput)
  255. avatarName = checkInput
  256. }
  257. if avatarName != NullUUID && avatarKey != NullUUID {
  258. fmt.Println(avatarName, "which has UUID:", avatarKey, "comes from grid:", gridName)
  259. } else {
  260. fmt.Println("Sorry, unknown input", checkInput)
  261. }
  262. }
  263. // never leaves until Ctrl-C
  264. }
  265. // set up routing.
  266. // NOTE(gwyneth): one function only because FastCGI seems to have problems with multiple handlers.
  267. http.HandleFunc("/", handler)
  268. log.Info("Directory for database:", *goslConfig.myDir)
  269. if (*goslConfig.isServer) {
  270. log.Info("Starting to run as web server on port " + *goslConfig.myPort)
  271. err := http.ListenAndServe(":" + *goslConfig.myPort, nil) // set listen port
  272. checkErrPanic(err) // if it can't listen to all the above, then it has to abort anyway
  273. } else {
  274. // default is to run as FastCGI!
  275. // works like a charm thanks to http://www.dav-muz.net/blog/2013/09/how-to-use-go-and-fastcgi/
  276. log.Debug("http.DefaultServeMux is", http.DefaultServeMux)
  277. if err := fcgi.Serve(nil, nil); err != nil {
  278. checkErrPanic(err)
  279. }
  280. }
  281. // we should never have reached this point!
  282. log.Error("Unknown usage! This application may run as a standalone server, as FastCGI application, or as an interactive shell")
  283. if *goslConfig.isServer || *goslConfig.isShell {
  284. flag.PrintDefaults()
  285. }
  286. }
  287. // handler deals with incoming queries and/or associates avatar names with keys depending on parameters.
  288. // Basically we check if both an avatar name and a UUID key has been received: if yes, this means a new entry;
  289. // if just the avatar name was received, it means looking up its key;
  290. // if just the key was received, it means looking up the name (not necessary since llKey2Name does that, but it's just to illustrate);
  291. // if nothing is received, then return an error
  292. func handler(w http.ResponseWriter, r *http.Request) {
  293. if err := r.ParseForm(); err != nil {
  294. logErrHTTP(w, http.StatusNotFound, "No avatar and/or UUID received")
  295. return
  296. }
  297. // test first if this comes from Second Life or OpenSimulator
  298. /*
  299. if r.Header.Get("X-Secondlife-Region") == "" {
  300. logErrHTTP(w, http.StatusForbidden, "Sorry, this application only works inside Second Life.")
  301. return
  302. }
  303. */
  304. name := r.Form.Get("name") // can be empty
  305. key := r.Form.Get("key") // can be empty
  306. compat := r.Form.Get("compat") // compatibility mode with W-Hat
  307. var valueToInsert avatarUUID
  308. messageToSL := "" // this is what we send back to SL - defined here due to scope issues.
  309. if name != "" {
  310. if key != "" {
  311. // we received both: add a new entry
  312. valueToInsert.UUID = key
  313. valueToInsert.Grid = r.Header.Get("X-Secondlife-Shard")
  314. jsonValueToInsert, err := json.Marshal(valueToInsert)
  315. checkErr(err)
  316. if *goslConfig.database == "badger" {
  317. kv, err := badger.Open(Opt)
  318. checkErrPanic(err) // should probably panic
  319. txn := kv.NewTransaction(true)
  320. defer txn.Discard()
  321. err = txn.Set([]byte(name), jsonValueToInsert)
  322. checkErrPanic(err)
  323. err = txn.Commit(nil)
  324. checkErrPanic(err)
  325. kv.Close()
  326. } else if *goslConfig.database == "buntdb" {
  327. db, err := buntdb.Open(goslConfig.dbNamePath)
  328. checkErrPanic(err)
  329. defer db.Close()
  330. err = db.Update(func(tx *buntdb.Tx) error {
  331. _, _, err := tx.Set(name, string(jsonValueToInsert), nil)
  332. return err
  333. })
  334. checkErr(err)
  335. } else if *goslConfig.database == "leveldb" {
  336. db, err := leveldb.OpenFile(goslConfig.dbNamePath, nil)
  337. checkErrPanic(err)
  338. err = db.Put([]byte(name), jsonValueToInsert, nil)
  339. checkErrPanic(err)
  340. db.Close()
  341. }
  342. messageToSL += "Added new entry for '" + name + "' which is: " + valueToInsert.UUID + " from grid: '" + valueToInsert.Grid + "'"
  343. } else {
  344. // we received a name: look up its UUID key and grid.
  345. key, grid := searchKVname(name)
  346. if compat == "false" {
  347. messageToSL += "UUID for '" + name + "' is: " + key + " from grid: '" + grid + "'"
  348. } else { // empty also means true!
  349. messageToSL += key
  350. }
  351. }
  352. } else if key != "" {
  353. // in this scenario, we have the UUID key but no avatar name: do the equivalent of a llKey2Name (slow)
  354. name, grid := searchKVUUID(key)
  355. if compat == "false" {
  356. messageToSL += "Avatar name for " + key + "' is '" + name + "' on grid: '" + grid + "'"
  357. } else { // empty also means true!
  358. messageToSL += name
  359. }
  360. } else {
  361. // neither UUID key nor avatar received, this is an error
  362. logErrHTTP(w, http.StatusNotFound, "Empty avatar name and UUID key received, cannot proceed")
  363. return
  364. }
  365. w.WriteHeader(http.StatusOK)
  366. w.Header().Set("Content-Type", "text/plain; charset=utf-8")
  367. fmt.Fprintf(w, messageToSL)
  368. }
  369. // searchKVname searches the KV database for an avatar name.
  370. func searchKVname(avatarName string) (UUID string, grid string) {
  371. var val = avatarUUID{ NullUUID, "" }
  372. time_start := time.Now()
  373. var err error // to deal with scope issues
  374. if *goslConfig.database == "badger" {
  375. kv, err := badger.Open(Opt)
  376. checkErrPanic(err)
  377. defer kv.Close()
  378. err = kv.View(func(txn *badger.Txn) error {
  379. item, err := txn.Get([]byte(avatarName))
  380. if err != nil {
  381. return err
  382. }
  383. data, err := item.Value()
  384. if err != nil {
  385. log.Errorf("Error '%s' while getting data from %v\n", err, item)
  386. return err
  387. }
  388. err = json.Unmarshal(data, &val)
  389. if err != nil {
  390. log.Errorf("Error while unparsing UUID for name: '%s' (%v)\n", avatarName, err)
  391. return err
  392. }
  393. return nil
  394. })
  395. } else if *goslConfig.database == "buntdb" {
  396. db, err := buntdb.Open(goslConfig.dbNamePath)
  397. checkErrPanic(err)
  398. defer db.Close()
  399. var data string
  400. err = db.View(func(tx *buntdb.Tx) error {
  401. data, err = tx.Get(avatarName)
  402. return err
  403. })
  404. err = json.Unmarshal([]byte(data), &val)
  405. if err != nil {
  406. log.Errorf("Error while unparsing UUID for name: '%s' (%v)\n", avatarName, err)
  407. }
  408. } else if *goslConfig.database == "leveldb" {
  409. db, err := leveldb.OpenFile(goslConfig.dbNamePath, nil)
  410. checkErrPanic(err)
  411. defer db.Close()
  412. data, err := db.Get([]byte(avatarName), nil)
  413. if err != nil {
  414. log.Errorf("Error while getting UUID for name: '%s' (%v)\n", avatarName, err)
  415. } else {
  416. err = json.Unmarshal(data, &val)
  417. if err != nil {
  418. log.Errorf("Error while unparsing UUID for name: '%s' (%v)\n", avatarName, err)
  419. }
  420. }
  421. }
  422. log.Debugf("Time to lookup '%s': %v\n", avatarName, time.Since(time_start))
  423. if err != nil {
  424. return NullUUID, ""
  425. } // else:
  426. return val.UUID, val.Grid
  427. }
  428. // searchKVUUID searches the KV database for an avatar key.
  429. func searchKVUUID(avatarKey string) (name string, grid string) {
  430. time_start := time.Now()
  431. checks := 0
  432. var val = avatarUUID{ NullUUID, "" }
  433. var found string
  434. if *goslConfig.database == "badger" {
  435. kv, err := badger.Open(Opt)
  436. checkErr(err) // should probably panic
  437. itOpt := badger.DefaultIteratorOptions
  438. /*
  439. if !*goslConfig.noMemory {
  440. itOpt.PrefetchValues = true
  441. itOpt.PrefetchSize = 1000 // attempt to get this a little bit more efficient; we have many small entries, so this is not too much
  442. } else {
  443. */
  444. itOpt.PrefetchValues = false // allegedly this is supposed to be WAY faster...
  445. // }
  446. txn := kv.NewTransaction(true)
  447. defer txn.Discard()
  448. err = kv.View(func(txn *badger.Txn) error {
  449. itr := txn.NewIterator(itOpt)
  450. defer itr.Close()
  451. for itr.Rewind(); itr.Valid(); itr.Next() {
  452. item := itr.Item()
  453. data, err := item.Value()
  454. if err != nil {
  455. log.Errorf("Error '%s' while getting data from %v\n", err, item)
  456. return err
  457. }
  458. err = json.Unmarshal(data, &val)
  459. if err != nil {
  460. log.Errorf("Error '%s' while unparsing UUID for data: %v\n", err, data)
  461. return err
  462. }
  463. checks++ //Just to see how many checks we made, for statistical purposes
  464. if avatarKey == val.UUID {
  465. found = string(item.Key())
  466. break
  467. }
  468. }
  469. return nil
  470. })
  471. kv.Close()
  472. } else if *goslConfig.database == "buntdb" {
  473. db, err := buntdb.Open(goslConfig.dbNamePath)
  474. checkErrPanic(err)
  475. err = db.View(func(tx *buntdb.Tx) error {
  476. err := tx.Ascend("", func(key, value string) bool {
  477. err = json.Unmarshal([]byte(value), &val)
  478. if err != nil {
  479. log.Errorf("Error '%s' while unparsing UUID for value: %v\n", err, value)
  480. }
  481. checks++ //Just to see how many checks we made, for statistical purposes
  482. if avatarKey == val.UUID {
  483. found = key
  484. return false
  485. }
  486. return true
  487. })
  488. return err
  489. })
  490. db.Close()
  491. } else if *goslConfig.database == "leveldb" {
  492. db, err := leveldb.OpenFile(goslConfig.dbNamePath, nil)
  493. checkErrPanic(err)
  494. iter := db.NewIterator(nil, nil)
  495. for iter.Next() {
  496. // Remember that the contents of the returned slice should not be modified, and
  497. // only valid until the next call to Next.
  498. key := iter.Key()
  499. value := iter.Value()
  500. err = json.Unmarshal(value, &val)
  501. if err != nil {
  502. log.Errorf("Error '%s' while unparsing UUID for data: %v\n", err, value)
  503. continue // a bit insane, but at least we will skip a few broken records
  504. }
  505. checks++ //Just to see how many checks we made, for statistical purposes
  506. if avatarKey == val.UUID {
  507. found = string(key)
  508. break
  509. }
  510. }
  511. iter.Release()
  512. err = iter.Error()
  513. db.Close()
  514. }
  515. log.Debugf("Made %d checks for '%s' in %v\n", checks, avatarKey, time.Since(time_start))
  516. return found, val.Grid
  517. }
  518. // importDatabase is essentially reading a bzip2'ed CSV file with UUID,AvatarName downloaded from http://w-hat.com/#name2key .
  519. // One could theoretically set a cron job to get this file, save it on disk periodically, and keep the database up-to-date
  520. // see https://stackoverflow.com/questions/24673335/how-do-i-read-a-gzipped-csv-file for the actual usage of these complicated things!
  521. func importDatabase(filename string) {
  522. filehandler, err := os.Open(filename)
  523. if err != nil {
  524. log.Fatal(err)
  525. }
  526. defer filehandler.Close()
  527. gr := bzip2.NewReader(filehandler) // open bzip2 reader
  528. cr := csv.NewReader(gr) // open csv reader and feed the bzip2 reader into it
  529. limit := 0 // outside of for loop so that we can count how many entries we had in total
  530. time_start := time.Now() // we want to get an idea on how long this takes
  531. if *goslConfig.database == "badger" {
  532. // prepare connection to KV database
  533. kv, err := badger.Open(Opt)
  534. checkErrPanic(err) // should probably panic
  535. defer kv.Close()
  536. txn := kv.NewTransaction(true) // start new transaction; we will commit only every BATCH_BLOCK entries
  537. defer txn.Discard()
  538. for ;;limit++ {
  539. record, err := cr.Read()
  540. if err == io.EOF {
  541. break
  542. } else if err != nil {
  543. log.Fatal(err)
  544. }
  545. jsonNewEntry, err := json.Marshal(avatarUUID{ record[0], "Production" }) // W-Hat keys come all from the main LL grid, known as 'Production'
  546. if err != nil {
  547. log.Warning(err)
  548. } else {
  549. err = txn.Set([]byte(record[1]), jsonNewEntry)
  550. if err != nil {
  551. log.Fatal(err)
  552. }
  553. }
  554. if limit % goslConfig.BATCH_BLOCK == 0 && limit != 0 { // we do not run on the first time, and then only every BATCH_BLOCK times
  555. log.Info("Processing:", limit)
  556. err = txn.Commit(nil)
  557. if err != nil {
  558. log.Fatal(err)
  559. }
  560. runtime.GC()
  561. txn = kv.NewTransaction(true) // start a new transaction
  562. defer txn.Discard()
  563. }
  564. }
  565. // commit last batch
  566. err = txn.Commit(nil)
  567. if err != nil {
  568. log.Fatal(err)
  569. }
  570. kv.PurgeOlderVersions()
  571. } else if *goslConfig.database == "buntdb" {
  572. db, err := buntdb.Open(goslConfig.dbNamePath)
  573. checkErrPanic(err)
  574. defer db.Close()
  575. txn, err := db.Begin(true)
  576. checkErrPanic(err)
  577. //defer txn.Commit()
  578. // very similar to Badger code...
  579. for ;;limit++ {
  580. record, err := cr.Read()
  581. if err == io.EOF {
  582. break
  583. } else if err != nil {
  584. log.Fatal(err)
  585. }
  586. jsonNewEntry, err := json.Marshal(avatarUUID{ record[0], "Production" })
  587. if err != nil {
  588. log.Warning(err)
  589. } else {
  590. _, _, err = txn.Set(record[1], string(jsonNewEntry), nil)
  591. if err != nil {
  592. log.Fatal(err)
  593. }
  594. }
  595. if limit % goslConfig.BATCH_BLOCK == 0 && limit != 0 { // we do not run on the first time, and then only every BATCH_BLOCK times
  596. log.Info("Processing:", limit)
  597. err = txn.Commit()
  598. if err != nil {
  599. log.Fatal(err)
  600. }
  601. runtime.GC()
  602. txn, err = db.Begin(true) // start a new transaction
  603. checkErrPanic(err)
  604. //defer txn.Commit()
  605. }
  606. }
  607. // commit last batch
  608. err = txn.Commit()
  609. if err != nil {
  610. log.Fatal(err)
  611. }
  612. db.Shrink()
  613. } else if *goslConfig.database == "leveldb" {
  614. db, err := leveldb.OpenFile(goslConfig.dbNamePath, nil)
  615. checkErrPanic(err)
  616. defer db.Close()
  617. batch := new(leveldb.Batch)
  618. for ;;limit++ {
  619. record, err := cr.Read()
  620. if err == io.EOF {
  621. break
  622. } else if err != nil {
  623. log.Fatal(err)
  624. }
  625. jsonNewEntry, err := json.Marshal(avatarUUID{ record[0], "Production" })
  626. if err != nil {
  627. log.Warning(err)
  628. } else {
  629. batch.Put([]byte(record[1]), jsonNewEntry)
  630. }
  631. if limit % goslConfig.BATCH_BLOCK == 0 && limit != 0 {
  632. log.Info("Processing:", limit)
  633. err = db.Write(batch, nil)
  634. if err != nil {
  635. log.Fatal(err)
  636. }
  637. batch.Reset() // unlike the others, we don't need to create a new batch every time
  638. runtime.GC() // it never hurts...
  639. }
  640. }
  641. // commit last batch
  642. err = db.Write(batch, nil)
  643. if err != nil {
  644. log.Fatal(err)
  645. }
  646. batch.Reset() // reset it and let the garbage collector run
  647. runtime.GC()
  648. db.CompactRange(util.Range{ nil, nil })
  649. }
  650. log.Info("Total read", limit, "records (or thereabouts) in", time.Since(time_start))
  651. }
  652. // NOTE(gwyneth): Auxiliary functions which I'm always using...
  653. // checkErrPanic logs a fatal error and panics.
  654. func checkErrPanic(err error) {
  655. if err != nil {
  656. pc, file, line, ok := runtime.Caller(1)
  657. log.Panic(filepath.Base(file), ":", line, ":", pc, ok, " - panic:", err)
  658. }
  659. }
  660. // checkErr checks if there is an error, and if yes, it logs it out and continues.
  661. // this is for 'normal' situations when we want to get a log if something goes wrong but do not need to panic
  662. func checkErr(err error) {
  663. if err != nil {
  664. pc, file, line, ok := runtime.Caller(1)
  665. log.Error(filepath.Base(file), ":", line, ":", pc, ok, " - error:", err)
  666. }
  667. }
  668. // Auxiliary functions for HTTP handling
  669. // checkErrHTTP returns an error via HTTP and also logs the error.
  670. func checkErrHTTP(w http.ResponseWriter, httpStatus int, errorMessage string, err error) {
  671. if err != nil {
  672. http.Error(w, fmt.Sprintf(errorMessage, err), httpStatus)
  673. pc, file, line, ok := runtime.Caller(1)
  674. log.Error("(", http.StatusText(httpStatus), ") ", filepath.Base(file), ":", line, ":", pc, ok, " - error:", errorMessage, err)
  675. }
  676. }
  677. // checkErrPanicHTTP returns an error via HTTP and logs the error with a panic.
  678. func checkErrPanicHTTP(w http.ResponseWriter, httpStatus int, errorMessage string, err error) {
  679. if err != nil {
  680. http.Error(w, fmt.Sprintf(errorMessage, err), httpStatus)
  681. pc, file, line, ok := runtime.Caller(1)
  682. log.Panic("(", http.StatusText(httpStatus), ") ", filepath.Base(file), ":", line, ":", pc, ok, " - panic:", errorMessage, err)
  683. }
  684. }
  685. // logErrHTTP assumes that the error message was already composed and writes it to HTTP and logs it.
  686. // this is mostly to avoid code duplication and make sure that all entries are written similarly
  687. func logErrHTTP(w http.ResponseWriter, httpStatus int, errorMessage string) {
  688. http.Error(w, errorMessage, httpStatus)
  689. log.Error("(" + http.StatusText(httpStatus) + ") " + errorMessage)
  690. }
  691. // funcName is @Sonia's solution to get the name of the function that Go is currently running.
  692. // This will be extensively used to deal with figuring out where in the code the errors are!
  693. // Source: https://stackoverflow.com/a/10743805/1035977 (20170708)
  694. func funcName() string {
  695. pc, _, _, _ := runtime.Caller(1)
  696. return runtime.FuncForPC(pc).Name()
  697. }
  698. // isValidUUID checks if the UUID is valid.
  699. // Thanks to Patrick D'Appollonio https://stackoverflow.com/questions/25051675/how-to-validate-uuid-v4-in-go
  700. func isValidUUID(uuid string) bool {
  701. r := regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}$")
  702. return r.MatchString(uuid)
  703. }