cache roms, multithreaded hashing
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Steven Polley 2023-06-04 12:32:18 -06:00
parent 3e5935bbcf
commit 0710bbea5d

100
main.go
View File

@ -11,6 +11,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync"
) )
// Information for a LineageOS ROM available for download // Information for a LineageOS ROM available for download
@ -25,10 +26,25 @@ type LineageOSROM struct {
} }
// The HTTP response JSON should be a JSON array of lineageOSROMS available for download // The HTTP response JSON should be a JSON array of lineageOSROMS available for download
type HttpResponseJSON struct { type HTTPResponseJSON struct {
Response []LineageOSROM `json:"response"` Response []LineageOSROM `json:"response"`
} }
type ROMCache struct {
ROMs []LineageOSROM
Cached map[string]bool // to quickly lookup if a file is already cached
sync.Mutex
}
var (
romCache ROMCache
)
// Preload cached list of files and hashes
func init() {
go updateROMCache("public")
}
// HTTP Routing // HTTP Routing
func main() { func main() {
@ -44,73 +60,85 @@ func main() {
} }
// Reads the ROM files on the filesystem and populates a slice of linageOSROMs // Reads the ROM files on the filesystem and populates a slice of linageOSROMs
func getLineageOSROMs(romDirectory string) ([]LineageOSROM, error) { func updateROMCache(romDirectory string) {
if _, err := os.Stat(romDirectory); os.IsNotExist(err) { if _, err := os.Stat(romDirectory); os.IsNotExist(err) {
return nil, fmt.Errorf("romDirectory '%s' does not exist", romDirectory) log.Printf("romDirectory '%s' does not exist", romDirectory)
return
} }
wg := sync.WaitGroup{}
// Get all the zip files and populate romFileNames slice
var lineageOSROMs []LineageOSROM
err := filepath.WalkDir(romDirectory, func(s string, d fs.DirEntry, err error) error { err := filepath.WalkDir(romDirectory, func(s string, d fs.DirEntry, err error) error {
if err != nil { if err != nil {
return fmt.Errorf("walk error occured during file '%s': %v", d.Name(), err) return fmt.Errorf("walk error occured during file '%s': %v", d.Name(), err)
} }
if filepath.Ext(d.Name()) != ".zip" { if filepath.Ext(d.Name()) != ".zip" {
return nil return nil
} }
fInfo, err := d.Info() // skip already cached files
if err != nil { romCache.Lock()
return fmt.Errorf("failed to get file info '%s': %v", d.Name(), err) if romCache.Cached[d.Name()] {
romCache.Unlock()
return nil
} }
romCache.Unlock()
// Get information about file and populate rom // Get information about file and populate rom
splitName := strings.Split(d.Name(), "-") splitName := strings.Split(d.Name(), "-")
if len(splitName) != 5 { if len(splitName) != 5 {
log.Printf("ignoring zip file '%s', name is not formatted correctly", d.Name()) log.Printf("ignoring zip file '%s', name is not formatted correctly", d.Name())
return nil return nil
} }
fileHash, err := hashFile(fmt.Sprintf("%s/%s", romDirectory, d.Name())) wg.Add(1)
if err != nil { go func(d fs.DirEntry, wg *sync.WaitGroup) {
log.Printf("ingore zip file '%s', failed to get sha256 hash: %v", d.Name(), err) defer wg.Done()
} fInfo, err := d.Info()
if err != nil {
log.Printf("failed to get file info '%s': %v", d.Name(), err)
return
}
lineageOSROM := LineageOSROM{ fileHash, err := hashFile(fmt.Sprintf("%s/%s", romDirectory, d.Name()))
Datetime: int(fInfo.ModTime().Unix()), if err != nil {
Filename: d.Name(), log.Printf("ingore zip file '%s', failed to get sha256 hash: %v", d.Name(), err)
ID: fileHash, return
Romtype: "nightly", }
Size: int(fInfo.Size()),
URL: fmt.Sprintf("https://lineageos-ota.deadbeef.codes/public/%s", d.Name()),
Version: splitName[1],
}
lineageOSROMs = append(lineageOSROMs, lineageOSROM) lineageOSROM := LineageOSROM{
Datetime: int(fInfo.ModTime().Unix()),
Filename: d.Name(),
ID: fileHash,
Romtype: "nightly",
Size: int(fInfo.Size()),
URL: fmt.Sprintf("https://lineageos-ota.deadbeef.codes/public/%s", d.Name()),
Version: splitName[1],
}
romCache.Lock()
romCache.ROMs = append(romCache.ROMs, lineageOSROM)
romCache.Cached[d.Name()] = true
romCache.Unlock()
}(d, &wg)
return nil return nil
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to walk romDirectory '%s': %v", romDirectory, err) log.Printf("failed to walk romDirectory '%s': %v", romDirectory, err)
return
} }
return lineageOSROMs, nil
} }
// http - GET / // http - GET /
// Writes JSON response for the updater app to know what versions are available to download // Writes JSON response for the updater app to know what versions are available to download
func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) { func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
lineageOSROMs, err := getLineageOSROMs("public") romCache.Lock()
if err != nil { lineageOSROMs := romCache.ROMs
w.WriteHeader(http.StatusInternalServerError) romCache.Unlock()
log.Printf("failed to get lineageOSROMs: %v", err)
return
}
httpResponseJSON := &HttpResponseJSON{Response: lineageOSROMs} httpResponseJSON := &HTTPResponseJSON{Response: lineageOSROMs}
b, err := json.Marshal(httpResponseJSON) b, err := json.Marshal(httpResponseJSON)
if err != nil { if err != nil {
@ -120,6 +148,8 @@ func lineageOSROMListHandler(w http.ResponseWriter, r *http.Request) {
} }
w.Write(b) w.Write(b)
go updateROMCache("public")
} }
func hashFile(filename string) (string, error) { func hashFile(filename string) (string, error) {
@ -134,5 +164,5 @@ func hashFile(filename string) (string, error) {
return "", fmt.Errorf("failed to copy data from file to hash function: %v", err) return "", fmt.Errorf("failed to copy data from file to hash function: %v", err)
} }
return string(h.Sum(nil)), nil return fmt.Sprintf("%x", h.Sum(nil)), nil
} }