cache roms, multithreaded hashing

main.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 )
 
 // Information for a LineageOS ROM available for download
@@ -25,10 +26,25 @@ type LineageOSROM struct {
 }
 
 // The HTTP response JSON should be a JSON array of lineageOSROMS available for download
-type HttpResponseJSON struct {
+type HTTPResponseJSON struct {
 	Response []LineageOSROM `json:"response"`
 }
 
+type ROMCache struct {
+	ROMs   []LineageOSROM
+	Cached map[string]bool // to quickly lookup if a file is already cached
+	sync.Mutex
+}
+
+var (
+	romCache ROMCache
+)
+
+// Preload cached list of files and hashes
+func init() {
+	go updateROMCache("public")
+}
+
 // HTTP Routing
 func main() {
 
@@ -44,73 +60,85 @@ func main() {
 }
 
 // Reads the ROM files on the filesystem and populates a slice of linageOSROMs
-func getLineageOSROMs(romDirectory string) ([]LineageOSROM, error) {
+func updateROMCache(romDirectory string) {
 
 	if _, err := os.Stat(romDirectory); os.IsNotExist(err) {
-		return nil, fmt.Errorf("romDirectory '%s' does not exist", romDirectory)
+		log.Printf("romDirectory '%s' does not exist", romDirectory)
+		return
 	}
-
+	wg := sync.WaitGroup{}
-	// Get all the zip files and populate romFileNames slice
-	var lineageOSROMs []LineageOSROM
 
 	err := filepath.WalkDir(romDirectory, func(s string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return fmt.Errorf("walk error occured during file '%s': %v", d.Name(), err)
+
 		}
 		if filepath.Ext(d.Name()) != ".zip" {
 			return nil
 		}
 
-		fInfo, err := d.Info()
+		// skip already cached files
-		if err != nil {
+		romCache.Lock()
-			return fmt.Errorf("failed to get file info '%s': %v", d.Name(), err)
+		if romCache.Cached[d.Name()] {
+			romCache.Unlock()
+			return nil
 		}
+		romCache.Unlock()
 
 		// Get information about file and populate rom
-
 		splitName := strings.Split(d.Name(), "-")
 		if len(splitName) != 5 {
 			log.Printf("ignoring zip file '%s', name is not formatted correctly", d.Name())
 			return nil
 		}
 
-		fileHash, err := hashFile(fmt.Sprintf("%s/%s", romDirectory, d.Name()))
+		wg.Add(1)
-		if err != nil {
+		go func(d fs.DirEntry, wg *sync.WaitGroup) {
-			log.Printf("ingore zip file '%s', failed to get sha256 hash: %v", d.Name(), err)
+			defer wg.Done()
-		}
+			fInfo, err := d.Info()
+			if err != nil {
+				log.Printf("failed to get file info '%s': %v", d.Name(), err)
+				return
+			}
 
-		lineageOSROM := LineageOSROM{
+			fileHash, err := hashFile(fmt.Sprintf("%s/%s", romDirectory, d.Name()))
-			Datetime: int(fInfo.ModTime().Unix()),
+			if err != nil {
-			Filename: d.Name(),
+				log.Printf("ingore zip file '%s', failed to get sha256 hash: %v", d.Name(), err)
-			ID:       fileHash,
+				return
-			Romtype:  "nightly",
+			}
-			Size:     int(fInfo.Size()),
-			URL:      fmt.Sprintf("https://lineageos-ota.deadbeef.codes/public/%s", d.Name()),
-			Version:  splitName[1],
-		}
 
-		lineageOSROMs = append(lineageOSROMs, lineageOSROM)
+			lineageOSROM := LineageOSROM{
+				Datetime: int(fInfo.ModTime().Unix()),
+				Filename: d.Name(),
+				ID:       fileHash,
+				Romtype:  "nightly",
+				Size:     int(fInfo.Size()),
+				URL:      fmt.Sprintf("https://lineageos-ota.deadbeef.codes/public/%s", d.Name()),
+				Version:  splitName[1],
+			}
+
+			romCache.Lock()
+			romCache.ROMs = append(romCache.ROMs, lineageOSROM)
+			romCache.Cached[d.Name()] = true
+			romCache.Unlock()
+		}(d, &wg)
 
 		return nil
 	})
 	if err != nil {
-		return nil, fmt.Errorf("failed to walk romDirectory '%s': %v", romDirectory, err)
+		log.Printf("failed to walk romDirectory '%s': %v", romDirectory, err)
+		return
 	}
-
-	return lineageOSROMs, nil
 }
 
 // http - GET /
 // Writes JSON response for the updater app to know what versions are available to download
 func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
-	lineageOSROMs, err := getLineageOSROMs("public")
+	romCache.Lock()
-	if err != nil {
+	lineageOSROMs := romCache.ROMs
-		w.WriteHeader(http.StatusInternalServerError)
+	romCache.Unlock()
-		log.Printf("failed to get lineageOSROMs: %v", err)
-		return
-	}
 
-	httpResponseJSON := &HttpResponseJSON{Response: lineageOSROMs}
+	httpResponseJSON := &HTTPResponseJSON{Response: lineageOSROMs}
 
 	b, err := json.Marshal(httpResponseJSON)
 	if err != nil {
@@ -120,6 +148,8 @@ func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	w.Write(b)
+
+	go updateROMCache("public")
 }
 
 func hashFile(filename string) (string, error) {
@@ -134,5 +164,5 @@ func hashFile(filename string) (string, error) {
 		return "", fmt.Errorf("failed to copy data from file to hash function: %v", err)
 	}
 
-	return string(h.Sum(nil)), nil
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
 }