diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..f905252 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,30 @@ +name: Fafda Binary + +on: + release: + types: [created] + +jobs: + release-fafda: + name: Release fafda binary + permissions: + contents: write + packages: write + runs-on: ubuntu-latest + strategy: + matrix: + goos: [linux] + goarch: [amd64] + steps: + - uses: actions/checkout@v3 + - uses: wangyoucao577/go-release-action@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: ${{ matrix.goos }} + goarch: ${{ matrix.goarch }} + project_path: "./cmd/fafda" + binary_name: "fafda" + compress_assets: "OFF" + md5sum: true + ldflags: "-s -w" + asset_name: "fafda-${{ matrix.goos }}-${{ matrix.goarch }}" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5c47eb4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.test +*.out + +bin +config.yaml +fafda.db diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..59707df --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +BINARY_NAME=fafda + +export CGO_ENABLED=0 + +tidy: + go fmt ./... + go mod tidy -v + +audit: + go mod verify + go vet ./... + go run honnef.co/go/tools/cmd/staticcheck@latest -checks=all,-ST1000,-U1000 ./... + go run golang.org/x/vuln/cmd/govulncheck@latest ./... + go test -race -buildvcs -vet=off ./... + +test: + go test -v -race -buildvcs ./... + +build: + go build -ldflags="-s -w" -o ./bin/$(BINARY_NAME) ./cmd/fafda + +run: build + ./bin/$(BINARY_NAME) --debug + +clean: + rm -rf ./bin/* + +build-linux: + GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o bin/release/$(BINARY_NAME) ./cmd/fafda diff --git a/README.md b/README.md index 587b7d4..12b3223 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ -# fafda -Tasty Tasty Fafda!! +## Fafda + +Fafda Fafda Good Good!! diff --git a/cmd/fafda/main.go b/cmd/fafda/main.go new file mode 100644 index 0000000..8dcb26d --- /dev/null +++ b/cmd/fafda/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "flag" + "fmt" + "os" + "runtime" + "strings" + "time" + + zl "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "go.etcd.io/bbolt" + + "fafda/config" + "fafda/internal" + "fafda/internal/bolt" + "fafda/internal/filesystem" + "fafda/internal/ftp" + "fafda/internal/github" + "fafda/internal/http" +) + +const name = "fafda" + +var ( + debugMode = flag.Bool("debug", false, "enable debug logs") + showVersion = flag.Bool("version", false, "print version information and exit") + configFile = flag.String("config", "", "path to nefarious configuration file") + listReleases = flag.String("list-releases", "", "comma-separated list of GitHub tokens to fetch releases information") +) + +func main() { + flag.Parse() + + if *showVersion { + fmt.Printf("%s: %s\n", name, internal.Version()) + os.Exit(0) + } + + if *listReleases != "" { + tokens := strings.Split(*listReleases, ",") + github.ListReleases(tokens) + os.Exit(0) + } + + runtime.GOMAXPROCS(runtime.NumCPU()) + + log.Logger = zl.New(zl.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339}).With().Timestamp().Logger() + zl.SetGlobalLevel(zl.InfoLevel) + if *debugMode { + zl.SetGlobalLevel(zl.DebugLevel) + } + + var err error + var cfg *config.Config + if *configFile != "" { + cfg, err = config.New(*configFile) + } else { + cfg, err = config.New() + } + if err != nil { + log.Fatal().Err(err).Msgf("failed to load config") + } + + dbFile := cfg.DBFile + if dbFile == "" { + dbFile = name + ".db" + } + + db, err := bbolt.Open(dbFile, 0600, nil) + if err != nil { + log.Fatal().Err(err).Msgf("failed to open bolt") + } + + metafs, err := bolt.NewMetaFs(db) + if err != nil { + log.Fatal().Err(err).Msgf("failed to open bolt data provider") + } + + driver, err := github.NewDriver(cfg.GitHub, db) + if err != nil { + log.Fatal().Err(err).Msgf("failed to load github driver") + } + + fs := filesystem.New(driver, metafs) + + if cfg.HTTPServer.Addr != "" { + go func() { + if err := http.Serv(cfg.HTTPServer, fs); err != nil { + log.Fatal().Err(err).Msgf("failed to start http server") + } + }() + } + + if err := ftp.Serv(cfg.FTPServer, fs); err != nil { + log.Fatal().Err(err).Msgf("failed to start ftp server") + } +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..dde2521 --- /dev/null +++ b/config/config.go @@ -0,0 +1,68 @@ +package config + +import ( + "fmt" + + "github.com/knadh/koanf" + "github.com/knadh/koanf/parsers/yaml" + "github.com/knadh/koanf/providers/file" +) + +type GitHubRelease struct { + ReadOnly bool `koanf:"readOnly"` + Username string `koanf:"username"` + AuthToken string `koanf:"authToken"` + ReleaseId int `koanf:"releaseId"` + ReleaseTag string `koanf:"releaseTag"` + Repository string `koanf:"repository"` +} + +type GitHub struct { + PartSize int64 `koanf:"partSize"` + Concurrency int `koanf:"concurrency"` + Releases []GitHubRelease `koanf:"releases"` +} + +type FTPPortRange struct { + Start int `koanf:"start"` + End int `koanf:"end"` +} + +type FTPServer struct { + Addr string `koanf:"addr"` + Username string `koanf:"username"` + Password string `koanf:"password"` + PortRange *FTPPortRange `koanf:"portRange"` +} + +type HTTPServer struct { + Addr string `koanf:"addr"` +} + +type Config struct { + DBFile string `koanf:"dbFile"` + GitHub GitHub `koanf:"github"` + FTPServer FTPServer `koanf:"ftpServer"` + HTTPServer HTTPServer `koanf:"httpServer"` +} + +var k = koanf.New(".") +var defaultConfigPath = "config.yaml" + +func New(configFile ...string) (*Config, error) { + configFilePath := defaultConfigPath + if len(configFile) > 0 { + configFilePath = configFile[0] + } + + if err := k.Load(file.Provider(configFilePath), yaml.Parser()); err != nil { + return nil, fmt.Errorf("load config from path - path %s - %w", configFilePath, err) + } + + var cfg Config + if err := k.Unmarshal("", &cfg); err != nil { + return nil, err + } + + return &cfg, nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..0a1f0fe --- /dev/null +++ b/go.mod @@ -0,0 +1,29 @@ +module fafda + +go 1.23.4 + +require ( + github.com/fclairamb/ftpserverlib v0.25.0 + github.com/knadh/koanf v1.5.0 + github.com/matoous/go-nanoid/v2 v2.1.0 + github.com/rs/zerolog v1.33.0 + github.com/spf13/afero v1.11.0 + go.etcd.io/bbolt v1.3.11 +) + +require ( + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/fclairamb/go-log v0.5.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.58.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..604a7ce --- /dev/null +++ b/go.sum @@ -0,0 +1,432 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= +github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= +github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= +github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fclairamb/ftpserverlib v0.25.0 h1:swV2CK+WiN9KEkqkwNgGbSIfRoYDWNno41hoVtYwgfA= +github.com/fclairamb/ftpserverlib v0.25.0/go.mod h1:LIDqyiFPhjE9IuzTkntST8Sn8TaU6NRgzSvbMpdfRC4= +github.com/fclairamb/go-log v0.5.0 h1:Gz9wSamEaA6lta4IU2cjJc2xSq5sV5VYSB5w/SUHhVc= +github.com/fclairamb/go-log v0.5.0/go.mod h1:XoRO1dYezpsGmLLkZE9I+sHqpqY65p8JA+Vqblb7k40= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grandpanutz/fasthttp v1.60.0 h1:7BunOVuL5KfqEaGwQtdhywnjsD/xy5+NZY12XKt195E= +github.com/grandpanutz/fasthttp v1.60.0/go.mod h1:8c7B6dgQrceUEn+9ErvOrAgjl66AhJ2SpN4ndlzMpQI= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.13.0/go.mod h1:ZlVrynguJKcYr54zGaDbaL3fOvKC9m72FhPvA8T35KQ= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= +github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= +github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs= +github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/knadh/koanf v1.5.0 h1:q2TSd/3Pyc/5yP9ldIrSdIz26MCcyNQzW0pEAugLPNs= +github.com/knadh/koanf v1.5.0/go.mod h1:Hgyjp4y8v44hpZtPzs7JZfRAW5AhN7KfZcwv1RYggDs= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= +github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.58.0/go.mod h1:SYXvHHaFp7QZHGKSHmoMipInhrI5StHrhDTYVEjK/Kw= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= +go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/bolt/bolt.go b/internal/bolt/bolt.go new file mode 100644 index 0000000..790b1ef --- /dev/null +++ b/internal/bolt/bolt.go @@ -0,0 +1,414 @@ +package bolt + +import ( + "bytes" + "encoding/gob" + "errors" + "fmt" + "os" + "path" + "strings" + "time" + + nanoid "github.com/matoous/go-nanoid/v2" + "go.etcd.io/bbolt" + + "fafda/internal" +) + +var fileBucket = []byte("files") + +type MetaFs struct { + db *bbolt.DB +} + +func NewMetaFs(db *bbolt.DB) (internal.MetaFileSystem, error) { + metafs := &MetaFs{db: db} + + err := db.Update(func(tx *bbolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists(fileBucket) + if err != nil { + return fmt.Errorf("failed to create file bucket %w", err) + } + + if data := bucket.Get([]byte("/")); data == nil { + root := NewFile("/", true) + if err := metafs.put(bucket, "/", root); err != nil { + return err + } + } + + return nil + }) + + if err != nil { + _ = db.Close() + return nil, err + } + + return metafs, nil +} + +func NewFile(path string, isDir bool) *internal.Node { + now := time.Now() + mode := os.FileMode(0644) + if isDir { + mode = os.FileMode(0755) | os.ModeDir + } + + node := &internal.Node{} + + if !isDir { + node.SetId(nanoid.Must()) + } + + return node. + SetPath(path). + SetIsDir(isDir). + SetSize(0). + SetMode(mode). + SetCreatedAt(now). + SetModTime(now) +} + +func (mf *MetaFs) Name() string { + return "boltdb" +} + +func (mf *MetaFs) get(bucket *bbolt.Bucket, path string) (*internal.Node, error) { + data := bucket.Get([]byte(path)) + if data == nil { + return nil, internal.ErrNotFound + } + return decodeNode(data) +} + +func (mf *MetaFs) put(bucket *bbolt.Bucket, path string, node *internal.Node) error { + data, err := encodeNode(node) + if err != nil { + return err + } + return bucket.Put([]byte(path), data) +} + +func (mf *MetaFs) checkParentDir(bucket *bbolt.Bucket, pathStr string) error { + parent := path.Dir(pathStr) + if parent == "/" { + return nil + } + + parentNode, err := mf.get(bucket, parent) + if err != nil { + return internal.ErrNotFound + } + + if !parentNode.IsDir() { + return internal.ErrNotFound + } + return nil +} + +func (mf *MetaFs) rename(tx *bbolt.Tx, b *bbolt.Bucket, data []byte, oldpath, newpath string) error { + node, err := decodeNode(data) + if err != nil { + return err + } + + node.SetPath(newpath) + if err := b.Delete([]byte(oldpath)); err != nil { + return err + } + + return mf.put(b, newpath, node) +} + +func (mf *MetaFs) Create(pathStr string, isDir bool) (*internal.Node, error) { + file := NewFile(pathStr, isDir) + + err := mf.db.Update(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(fileBucket) + + if bucket.Get([]byte(pathStr)) != nil { + return internal.ErrAlreadyExist + } + + if err := mf.checkParentDir(bucket, pathStr); err != nil { + return err + } + + return mf.put(bucket, pathStr, file) + }) + + if err != nil { + return nil, err + } + + return file, nil +} + +func (mf *MetaFs) Stat(path string) (*internal.Node, error) { + if path == "" || path == "/" { + return NewFile("/", true), nil + } + + var file *internal.Node + err := mf.db.View(func(tx *bbolt.Tx) error { + var err error + file, err = mf.get(tx.Bucket(fileBucket), path) + return err + }) + + return file, err +} + +func (mf *MetaFs) Ls(pathStr string, limit int, offset int) ([]internal.Node, error) { + info, err := mf.Stat(pathStr) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, internal.ErrIsNotDir + } + + var files []internal.Node + cleanPath := path.Clean(pathStr) + + var prefix []byte + if cleanPath == "/" { + prefix = []byte("/") + } else { + prefix = []byte(cleanPath + "/") + } + + err = mf.db.View(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(fileBucket) + c := bucket.Cursor() + + skipped := 0 + for k, v := c.Seek(prefix); k != nil; k, v = c.Next() { + if cleanPath == "/" { + if string(k) == "/" { + continue + } + parts := strings.Split(strings.TrimPrefix(string(k), "/"), "/") + if len(parts) > 1 { + continue + } + } else { + if !strings.HasPrefix(string(k), string(prefix)) { + break + } + relPath := strings.TrimPrefix(string(k), string(prefix)) + if strings.Contains(relPath, "/") { + continue + } + } + + if skipped < offset { + skipped++ + continue + } + + if limit != -1 && limit > 0 && len(files) >= limit { + break + } + + file, err := decodeNode(v) + if err != nil { + return err + } + files = append(files, *file) + } + return nil + }) + + return files, err +} + +func (mf *MetaFs) Chtimes(path string, mtime time.Time) error { + return mf.db.Update(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(fileBucket) + + node, err := mf.get(bucket, path) + if err != nil { + return err + } + + node.SetModTime(mtime) + return mf.put(bucket, path, node) + }) +} + +func (mf *MetaFs) Touch(path string) error { + _, err := mf.Stat(path) + if errors.Is(err, internal.ErrNotFound) { + _, err = mf.Create(path, false) + } + return err +} + +func (mf *MetaFs) Mkdir(path string) error { + _, err := mf.Create(path, true) + return err +} + +func (mf *MetaFs) MkdirAll(pathStr string) error { + pathStr = path.Clean(pathStr) + if pathStr == "/" { + return nil + } + + if _, err := mf.Stat(pathStr); err == nil { + return nil + } + + parent := path.Dir(pathStr) + if parent != "/" { + if err := mf.MkdirAll(parent); err != nil { + return err + } + } + + _, err := mf.Create(pathStr, true) + if err != nil && !errors.Is(err, internal.ErrAlreadyExist) { + return err + } + return nil +} + +func (mf *MetaFs) Rename(oldpath, newpath string) error { + oldpath = path.Clean(oldpath) + newpath = path.Clean(newpath) + + if oldpath == "/" { + return internal.ErrInvalidRootOperation + } + + if strings.HasPrefix(newpath, oldpath+"/") { + return internal.ErrInvalidOperation + } + + return mf.db.Update(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(fileBucket) + if exist := bucket.Get([]byte(newpath)); exist != nil { + return internal.ErrAlreadyExist + } + + data := bucket.Get([]byte(oldpath)) + if data == nil { + return internal.ErrNotFound + } + + if err := mf.checkParentDir(bucket, newpath); err != nil { + return err + } + + if err := mf.rename(tx, bucket, data, oldpath, newpath); err != nil { + return err + } + + // Handle renaming of child nodes + prefix := []byte(oldpath + "/") + newPrefix := []byte(newpath + "/") + c := bucket.Cursor() + var filesToMove [][]byte + for k, _ := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, _ = c.Next() { + filesToMove = append(filesToMove, k) + } + for _, f := range filesToMove { + newKey := append(newPrefix, f[len(prefix):]...) + if err := mf.rename(tx, bucket, bucket.Get(f), string(f), string(newKey)); err != nil { + return err + } + } + return nil + }) +} + +func (mf *MetaFs) Remove(path string) error { + return mf.db.Update(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(fileBucket) + + node, err := mf.get(bucket, path) + if err != nil { + return err + } + + if node.IsDir() { + prefix := path + "/" + c := bucket.Cursor() + k, _ := c.Seek([]byte(prefix)) + + if k != nil && strings.HasPrefix(string(k), prefix) { + return internal.ErrNotEmpty + } + } + + return bucket.Delete([]byte(path)) + }) +} + +func (mf *MetaFs) RemoveAll(pathStr string) error { + pathStr = path.Clean(pathStr) + if pathStr == "/" { + return internal.ErrInvalidRootOperation + } + + return mf.db.Update(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(fileBucket) + + if data := bucket.Get([]byte(pathStr)); data == nil { + return nil + } + + prefix := pathStr + "/" + c := bucket.Cursor() + + for k, _ := c.Seek([]byte(prefix)); k != nil && strings.HasPrefix(string(k), prefix); k, _ = c.Next() { + if err := bucket.Delete(k); err != nil { + return err + } + } + + return bucket.Delete([]byte(pathStr)) + }) +} + +func (mf *MetaFs) Sync(path string, size int64) error { + return mf.db.Update(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(fileBucket) + + node, err := mf.get(bucket, path) + if err != nil { + return err + } + + if !node.IsDir() { + node.SetSize(size) + } + node.SetModTime(time.Now()) + + return mf.put(bucket, path, node) + }) +} + +func (mf *MetaFs) Close() error { + return mf.db.Close() +} + +func encodeNode(node *internal.Node) ([]byte, error) { + var buf bytes.Buffer + if err := gob.NewEncoder(&buf).Encode(node); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func decodeNode(data []byte) (*internal.Node, error) { + var node internal.Node + buf := bytes.NewBuffer(data) + if err := gob.NewDecoder(buf).Decode(&node); err != nil { + return nil, err + } + return &node, nil +} diff --git a/internal/bolt/bolt_test.go b/internal/bolt/bolt_test.go new file mode 100644 index 0000000..804935d --- /dev/null +++ b/internal/bolt/bolt_test.go @@ -0,0 +1,1121 @@ +package bolt + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "go.etcd.io/bbolt" + + "fafda/internal" +) + +func setupTestDB(t *testing.T) (internal.MetaFileSystem, func()) { + tmpFile := filepath.Join(t.TempDir(), "test.db") + db, err := bbolt.Open(tmpFile, 0600, nil) + if err != nil { + t.Fatalf("failed to open bolt") + } + + provider, err := NewMetaFs(db) + if err != nil { + t.Fatalf("failed to create provider: %v", err) + } + + cleanup := func() { + _ = provider.Close() + _ = os.Remove(tmpFile) + } + + return provider, cleanup +} + +func TestCreate(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + tests := []struct { + name string + path string + isDir bool + wantErr error + }{ + { + name: "create directory", + path: "/test", + isDir: true, + wantErr: nil, + }, + { + name: "create file in directory", + path: "/test/file.txt", + isDir: false, + wantErr: nil, + }, + { + name: "create duplicate directory", + path: "/test", + isDir: true, + wantErr: internal.ErrAlreadyExist, + }, + { + name: "create duplicate file", + path: "/test/file.txt", + isDir: false, + wantErr: internal.ErrAlreadyExist, + }, + { + name: "create in missing directory", + path: "/missing/file.txt", + isDir: false, + wantErr: internal.ErrNotFound, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + file, err := provider.Create(tt.path, tt.isDir) + if tt.wantErr != nil { + if err == nil { + t.Error("expected error but got none") + return + } + if !errors.Is(err, tt.wantErr) { + t.Errorf("got error %q, want %q", err.Error(), tt.wantErr) + } + return + } + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + if file.IsDir() != tt.isDir { + t.Errorf("got isDir %v, want %v", file.IsDir(), tt.isDir) + } + if file.Name() != filepath.Base(tt.path) { + t.Errorf("got name %q, want %q", file.Name(), filepath.Base(tt.path)) + } + }) + } +} + +func TestStat(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + // Setup test directory and file + _, err := provider.Create("/test", true) + if err != nil { + t.Fatalf("setup failed: %v", err) + } + + _, err = provider.Create("/test/file.txt", false) + if err != nil { + t.Fatalf("setup failed: %v", err) + } + + tests := []struct { + name string + path string + wantDir bool + wantErr bool + }{ + { + name: "stat root", + path: "/", + wantDir: true, + wantErr: false, + }, + { + name: "stat directory", + path: "/test", + wantDir: true, + wantErr: false, + }, + { + name: "stat file", + path: "/test/file.txt", + wantDir: false, + wantErr: false, + }, + { + name: "stat non-existent", + path: "/missing", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info, err := provider.Stat(tt.path) + if tt.wantErr { + if err == nil { + t.Error("expected error but got none") + } + return + } + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + if info.IsDir() != tt.wantDir { + t.Errorf("got isDir %v, want %v", info.IsDir(), tt.wantDir) + } + }) + } +} + +func TestLs(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + // Setup test structure + _, err := provider.Create("/test", true) + if err != nil { + t.Fatalf("setup failed: %v", err) + } + + files := []string{"a.txt", "b.txt", "c.txt"} + for _, f := range files { + _, err := provider.Create("/test/"+f, false) + if err != nil { + t.Fatalf("setup failed: %v", err) + } + } + + tests := []struct { + name string + path string + limit int + offset int + wantCount int + wantErr bool + }{ + { + name: "list all files", + path: "/test", + limit: 0, + offset: 0, + wantCount: 3, + }, + { + name: "list with limit", + path: "/test", + limit: 2, + offset: 0, + wantCount: 2, + }, + { + name: "list with offset", + path: "/test", + limit: 0, + offset: 1, + wantCount: 2, + }, + { + name: "list non-existent directory", + path: "/missing", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + files, err := provider.Ls(tt.path, tt.limit, tt.offset) + if tt.wantErr { + if err == nil { + t.Error("expected error but got none") + } + return + } + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + if len(files) != tt.wantCount { + t.Errorf("got %d files, want %d", len(files), tt.wantCount) + } + }) + } +} + +func TestRemove(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + tests := []struct { + name string + setup func() error + cleanup func() error + path string + wantErr error + }{ + { + name: "remove file", + setup: func() error { + if _, err := provider.Create("/test", true); err != nil { + return err + } + if _, err := provider.Create("/test/file.txt", false); err != nil { + return err + } + return nil + }, + cleanup: func() error { + _ = provider.Remove("/test/file.txt") // Ignore errors as file might be already removed + _ = provider.Remove("/test") + return nil + }, + path: "/test/file.txt", + }, + { + name: "remove empty directory", + setup: func() error { + _, err := provider.Create("/empty", true) + return err + }, + cleanup: func() error { + _ = provider.Remove("/empty") + return nil + }, + path: "/empty", + }, + { + name: "remove non-empty directory", + setup: func() error { + if _, err := provider.Create("/test2", true); err != nil { + return err + } + if _, err := provider.Create("/test2/file.txt", false); err != nil { + return err + } + return nil + }, + cleanup: func() error { + _ = provider.Remove("/test2/file.txt") + _ = provider.Remove("/test2") + return nil + }, + path: "/test2", + wantErr: internal.ErrNotEmpty, + }, + { + name: "remove non-existent", + setup: func() error { return nil }, + cleanup: func() error { return nil }, + path: "/missing", + wantErr: internal.ErrNotFound, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.setup(); err != nil { + t.Fatalf("setup failed: %v", err) + } + + if info, err := provider.Stat(tt.path); err == nil && info.IsDir() { + files, err := provider.Ls(tt.path, 0, 0) + if err != nil { + t.Logf("Failed to list directory: %v", err) + } else { + t.Logf("Directory %s contains %d files", tt.path, len(files)) + for _, f := range files { + t.Logf("- %s", f.Name()) + } + } + } + + err := provider.Remove(tt.path) + + defer func() { + if err := tt.cleanup(); err != nil { + t.Logf("cleanup failed: %v", err) + } + }() + + if tt.wantErr != nil { + if err == nil { + t.Error("expected error but got none") + return + } + if !errors.Is(err, tt.wantErr) { + t.Errorf("got error %q, want %q", err.Error(), tt.wantErr) + } + return + } + if err != nil { + t.Errorf("unexpected error: %v", err) + } + + if _, err := provider.Stat(tt.path); err == nil { + t.Error("path still exists after removal") + } + }) + } +} + +func TestRename(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + tests := []struct { + name string + setup func() error + cleanup func() error + oldpath string + newpath string + wantErr error + }{ + { + name: "rename file", + setup: func() error { + if _, err := provider.Create("/test", true); err != nil { + return err + } + if _, err := provider.Create("/test/file.txt", false); err != nil { + return err + } + return nil + }, + cleanup: func() error { + _ = provider.Remove("/test/file.txt") + _ = provider.Remove("/test/newfile.txt") + _ = provider.Remove("/test") + return nil + }, + oldpath: "/test/file.txt", + newpath: "/test/newfile.txt", + }, + { + name: "rename directory", + setup: func() error { + if _, err := provider.Create("/dir1", true); err != nil { + return err + } + if _, err := provider.Create("/dir1/file.txt", false); err != nil { + return err + } + return nil + }, + cleanup: func() error { + _ = provider.Remove("/dir1/file.txt") + _ = provider.Remove("/dir1") + _ = provider.Remove("/dir2/file.txt") + _ = provider.Remove("/dir2") + return nil + }, + oldpath: "/dir1", + newpath: "/dir2", + }, + { + name: "rename non-existent", + setup: func() error { return nil }, + cleanup: func() error { return nil }, + oldpath: "/missing", + newpath: "/new", + wantErr: internal.ErrNotFound, + }, + { + name: "rename to existing path", + setup: func() error { + if _, err := provider.Create("/test2", true); err != nil { + return err + } + if _, err := provider.Create("/test2/file.txt", false); err != nil { + return err + } + return nil + }, + cleanup: func() error { + _ = provider.Remove("/test2/file.txt") + _ = provider.Remove("/test2") + return nil + }, + oldpath: "/test2/file.txt", + newpath: "/test2", + wantErr: internal.ErrAlreadyExist, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.setup(); err != nil { + t.Fatalf("setup failed: %v", err) + } + defer tt.cleanup() + + err := provider.Rename(tt.oldpath, tt.newpath) + if tt.wantErr != nil { + if err == nil { + t.Error("expected error but got none") + return + } + if !errors.Is(err, tt.wantErr) { + t.Errorf("got error %q, want %q", err.Error(), tt.wantErr) + } + return + } + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + if _, err := provider.Stat(tt.oldpath); err == nil { + t.Error("source still exists after rename") + } + + if _, err := provider.Stat(tt.newpath); err != nil { + t.Error("destination doesn't exist after rename") + } + }) + } +} + +func TestComplexRenameOperations(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + // Setup structure: + // /project + // /src + // /main.go + // /lib + // /util.go + // /test + // /main_test.go + // /docs + // /readme.md + + dirs := []string{ + "/project", + "/project/src", + "/project/src/lib", + "/project/test", + "/project/docs", + } + + files := []string{ + "/project/src/main.go", + "/project/src/lib/util.go", + "/project/test/main_test.go", + "/project/docs/readme.md", + } + + for _, dir := range dirs { + _, err := provider.Create(dir, true) + if err != nil { + t.Fatalf("failed to create dir %s: %v", dir, err) + } + } + + for _, file := range files { + _, err := provider.Create(file, false) + if err != nil { + t.Fatalf("failed to create file %s: %v", file, err) + } + } + + // 1. Move src/lib to src/utils + err := provider.Rename("/project/src/lib", "/project/src/utils") + if err != nil { + t.Fatalf("failed to rename lib to utils: %v", err) + } + + // 2. Move entire test directory into src + err = provider.Rename("/project/test", "/project/src/test") + if err != nil { + t.Fatalf("failed to move test into src: %v", err) + } + + // 3. Move src directory to root + err = provider.Rename("/project/src", "/src") + if err != nil { + t.Fatalf("failed to move src to root: %v", err) + } + + // Verify final structure + expectedPaths := []string{ + "/src/main.go", + "/src/utils/util.go", + "/src/test/main_test.go", + "/project/docs/readme.md", + } + + for _, path := range expectedPaths { + if _, err := provider.Stat(path); err != nil { + t.Errorf("expected path %s not found: %v", path, err) + } + } +} + +func TestMkdirAll(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + tests := []struct { + name string + path string + setup func() error + wantErr bool + }{ + { + name: "create single directory", + path: "/test", + }, + { + name: "create nested directories", + path: "/a/b/c/d", + }, + { + name: "create already existing directory", + path: "/existing", + setup: func() error { + _, err := provider.Create("/existing", true) + return err + }, + }, + { + name: "create with existing parent", + path: "/parent/child", + setup: func() error { + _, err := provider.Create("/parent", true) + return err + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + if err := tt.setup(); err != nil { + t.Fatalf("setup failed: %v", err) + } + } + + err := provider.MkdirAll(tt.path) + if (err != nil) != tt.wantErr { + t.Errorf("MkdirAll() error = %v, wantErr %v", err, tt.wantErr) + return + } + + // Verify directory exists + info, err := provider.Stat(tt.path) + if err != nil { + t.Errorf("failed to stat created directory: %v", err) + return + } + if !info.IsDir() { + t.Error("created path is not a directory") + } + + parent := filepath.Dir(tt.path) + for parent != "/" { + info, err := provider.Stat(parent) + if err != nil { + t.Errorf("parent directory %s not found: %v", parent, err) + } else if !info.IsDir() { + t.Errorf("parent path %s is not a directory", parent) + } + parent = filepath.Dir(parent) + } + }) + } +} + +func TestRemoveAll(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + tests := []struct { + name string + setup func() error + path string + wantErr bool + }{ + { + name: "remove single directory", + setup: func() error { + _, err := provider.Create("/test", true) + return err + }, + path: "/test", + }, + { + name: "remove nested structure", + setup: func() error { + if err := provider.MkdirAll("/a/b/c"); err != nil { + return err + } + if _, err := provider.Create("/a/b/c/file1.txt", false); err != nil { + return err + } + if _, err := provider.Create("/a/b/file2.txt", false); err != nil { + return err + } + return nil + }, + path: "/a", + }, + { + name: "remove root", + path: "/", + wantErr: true, + }, + { + name: "remove non-existent path", + path: "/missing", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + if err := tt.setup(); err != nil { + t.Fatalf("setup failed: %v", err) + } + } + + err := provider.RemoveAll(tt.path) + if (err != nil) != tt.wantErr { + t.Errorf("RemoveAll() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + if _, err := provider.Stat(tt.path); err == nil { + t.Error("path still exists after removal") + } + + files, err := provider.Ls(filepath.Dir(tt.path), 0, 0) + if err == nil { + for _, f := range files { + if strings.HasPrefix(f.Name(), filepath.Base(tt.path)) { + t.Errorf("found remaining file: %s", f.Name()) + } + } + } + } + }) + } +} + +func TestDeepDirectoryOperations(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + // /a/b/c/d/e/file.txt + path := "" + dirs := []string{"a", "b", "c", "d", "e"} + for _, dir := range dirs { + path = filepath.Join(path, dir) + fullPath := "/" + path + _, err := provider.Create(fullPath, true) + if err != nil { + t.Fatalf("failed to create %s: %v", fullPath, err) + } + } + + deepFile := "/a/b/c/d/e/file.txt" + _, err := provider.Create(deepFile, false) + if err != nil { + t.Fatalf("failed to create deep file: %v", err) + } + + // Move middle directory + // /a/b/c/d/e/file.txt -> /a/b/x/d/e/file.txt + err = provider.Rename("/a/b/c", "/a/b/x") + if err != nil { + t.Fatalf("failed to move middle directory: %v", err) + } + + _, err = provider.Stat("/a/b/x/d/e/file.txt") + if err != nil { + t.Error("file not found in new location") + } + + _, err = provider.Stat(deepFile) + if err == nil { + t.Error("old path still exists") + } +} + +func TestConcurrentOperations(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + // Create base directory first + _, err := provider.Create("/shared", true) + if err != nil { + t.Fatal(err) + } + + // Create all directories first + for i := 0; i < 10; i++ { + dirPath := fmt.Sprintf("/shared/dir%d", i) + _, err := provider.Create(dirPath, true) + if err != nil { + t.Fatalf("failed to create directory %s: %v", dirPath, err) + } + } + + var wg sync.WaitGroup + errs := make(chan error, 100) + + // Now create files concurrently + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + dirPath := fmt.Sprintf("/shared/dir%d", i) + // Create files in directory + for j := 0; j < 5; j++ { + filePath := fmt.Sprintf("%s/file%d.txt", dirPath, j) + _, err := provider.Create(filePath, false) + if err != nil { + errs <- fmt.Errorf("failed to create %s: %v", filePath, err) + return + } + } + }(i) + } + + wg.Wait() + + for i := 0; i < 5; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + oldPath := fmt.Sprintf("/shared/dir%d", i) + newPath := fmt.Sprintf("/shared/dir%d_renamed", i) + err := provider.Rename(oldPath, newPath) + if err != nil { + errs <- fmt.Errorf("failed to rename %s to %s: %v", oldPath, newPath, err) + } + }(i) + } + + wg.Wait() + close(errs) + + var testErrors []string + for err := range errs { + testErrors = append(testErrors, err.Error()) + } + if len(testErrors) > 0 { + t.Errorf("encountered errors:\n%s", strings.Join(testErrors, "\n")) + } + + files, err := provider.Ls("/shared", 0, 0) + if err != nil { + t.Fatal(err) + } + + if len(files) != 10 { + t.Errorf("expected 10 directories, got %d", len(files)) + } + + for i := 0; i < 5; i++ { + newPath := fmt.Sprintf("/shared/dir%d_renamed", i) + files, err := provider.Ls(newPath, 0, 0) + if err != nil { + t.Errorf("failed to list %s: %v", newPath, err) + continue + } + if len(files) != 5 { + t.Errorf("expected 5 files in %s, got %d", newPath, len(files)) + } + } +} + +func TestEdgeCases(t *testing.T) { + provider, cleanup := setupTestDB(t) + defer cleanup() + + cleanAll := func(t *testing.T) { + paths := []string{ + "/dir", + "/dir1", + "/dir2", + "/file.txt", + "/dir/file.txt", + "/dir/subfile", + "/missing", + } + for _, path := range paths { + _ = provider.Remove(path) + } + } + + tests := []struct { + name string + setup func(t *testing.T) + test func() error + wantErr error + }{ + { + name: "move directory into itself", + setup: func(t *testing.T) { + cleanAll(t) + _, err := provider.Create("/dir", true) + if err != nil { + t.Fatalf("create dir failed: %v", err) + } + _, err = provider.Create("/dir/file.txt", false) + if err != nil { + t.Fatalf("create file failed: %v", err) + } + }, + test: func() error { + return provider.Rename("/dir", "/dir/subdir") + }, + wantErr: internal.ErrInvalidOperation, + }, + { + name: "rename root directory", + setup: func(t *testing.T) { + cleanAll(t) + }, + test: func() error { + return provider.Rename("/", "/newroot") + }, + wantErr: internal.ErrInvalidRootOperation, + }, + { + name: "rename to existing path", + setup: func(t *testing.T) { + cleanAll(t) + _, err := provider.Create("/dir1", true) + if err != nil { + t.Fatalf("create dir1 failed: %v", err) + } + _, err = provider.Create("/dir2", true) + if err != nil { + t.Fatalf("create dir2 failed: %v", err) + } + }, + test: func() error { + return provider.Rename("/dir1", "/dir2") + }, + wantErr: internal.ErrAlreadyExist, + }, + { + name: "rename with missing parent", + setup: func(t *testing.T) { + cleanAll(t) + _, err := provider.Create("/dir", true) + if err != nil { + t.Fatalf("create dir failed: %v", err) + } + }, + test: func() error { + return provider.Rename("/dir", "/missing/dir") + }, + wantErr: internal.ErrNotFound, + }, + { + name: "rename to file path", + setup: func(t *testing.T) { + cleanAll(t) + _, err := provider.Create("/dir", true) + if err != nil { + t.Fatalf("create dir failed: %v", err) + } + _, err = provider.Create("/dir/subfile", false) + if err != nil { + t.Fatalf("create subfile failed: %v", err) + } + _, err = provider.Create("/file.txt", false) + if err != nil { + t.Fatalf("create file.txt failed: %v", err) + } + }, + test: func() error { + return provider.Rename("/dir", "/file.txt") + }, + wantErr: internal.ErrAlreadyExist, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.setup(t) + + err := tt.test() + if err == nil { + t.Error("expected error but got none") + return + } + if !errors.Is(err, tt.wantErr) { + t.Errorf("got error %q, want %q", err.Error(), tt.wantErr) + } + }) + } +} + +func TestVirtualFileSystemOperations(t *testing.T) { + fs, cleanup := setupTestDB(t) + defer cleanup() + + t.Run("1. Initial ls", func(t *testing.T) { + files, err := fs.Ls("/", 10, 0) + if err != nil { + t.Fatalf("Initial ls failed: %v", err) + } + if len(files) != 0 { + t.Error("Root directory should be empty") + } + }) + + t.Run("2. mkdir abc", func(t *testing.T) { + if err := fs.Mkdir("/abc"); err != nil { + t.Fatalf("Failed to create abc directory: %v", err) + } + + stat, err := fs.Stat("/abc") + if err != nil { + t.Fatalf("Failed to stat abc directory: %v", err) + } + if !stat.IsDir() { + t.Error("abc should be a directory") + } + }) + + t.Run("3. ls abc", func(t *testing.T) { + files, err := fs.Ls("/abc", 10, 0) + if err != nil { + t.Fatalf("Failed to list abc directory: %v", err) + } + if len(files) != 0 { + t.Error("abc directory should be empty") + } + }) + + t.Run("4. mkdir abc/hello", func(t *testing.T) { + if err := fs.Mkdir("/abc/hello"); err != nil { + t.Fatalf("Failed to create abc/hello directory: %v", err) + } + + stat, err := fs.Stat("/abc/hello") + if err != nil { + t.Fatalf("Failed to stat abc/hello directory: %v", err) + } + if !stat.IsDir() { + t.Error("abc/hello should be a directory") + } + }) + + t.Run("5. touch abc/hello/abc.txt", func(t *testing.T) { + _, err := fs.Create("/abc/hello/abc.txt", false) + if err != nil { + t.Fatalf("Failed to create abc/hello/abc.txt: %v", err) + } + + stat, err := fs.Stat("/abc/hello/abc.txt") + if err != nil { + t.Fatalf("Failed to stat abc/hello/abc.txt: %v", err) + } + if stat.IsDir() { + t.Error("abc.txt should be a file") + } + }) + + t.Run("6. rename abc/hello abc/xyz", func(t *testing.T) { + if err := fs.Rename("/abc/hello", "/abc/xyz"); err != nil { + t.Fatalf("Failed to rename hello to xyz: %v", err) + } + + if _, err := fs.Stat("/abc/hello"); err == nil { + t.Error("Old path should not exist") + } + + stat, err := fs.Stat("/abc/xyz") + if err != nil { + t.Fatalf("Failed to stat new path: %v", err) + } + if !stat.IsDir() { + t.Error("xyz should be a directory") + } + + if _, err := fs.Stat("/abc/xyz/abc.txt"); err != nil { + t.Fatalf("File should exist in new location: %v", err) + } + }) + + t.Run("7. rename abc/xyz/abc.txt abc/abc.txt", func(t *testing.T) { + if err := fs.Rename("/abc/xyz/abc.txt", "/abc/abc.txt"); err != nil { + t.Fatalf("Failed to move abc.txt: %v", err) + } + + stat, err := fs.Stat("/abc/abc.txt") + if err != nil { + t.Fatalf("Failed to stat moved file: %v", err) + } + if stat.IsDir() { + t.Error("abc.txt should be a file") + } + + if _, err := fs.Stat("/abc/xyz/abc.txt"); err == nil { + t.Error("File should not exist in old location") + } + }) + + t.Run("8. rename abc/xyz xyz", func(t *testing.T) { + if err := fs.Rename("/abc/xyz", "/xyz"); err != nil { + t.Fatalf("Failed to move xyz directory: %v", err) + } + + stat, err := fs.Stat("/xyz") + if err != nil { + t.Fatalf("Failed to stat moved directory: %v", err) + } + if !stat.IsDir() { + t.Error("xyz should be a directory") + } + + if _, err := fs.Stat("/abc/xyz"); err == nil { + t.Error("Directory should not exist in old location") + } + }) + + t.Run("9. sync abc/abc.txt 100", func(t *testing.T) { + if err := fs.Sync("/abc/abc.txt", 100); err != nil { + t.Fatalf("Failed to sync file size: %v", err) + } + + stat, err := fs.Stat("/abc/abc.txt") + if err != nil { + t.Fatalf("Failed to stat file: %v", err) + } + if stat.Size() != 100 { + t.Errorf("File size should be 100, got %d", stat.Size()) + } + }) + + t.Run("10. rename abc/abc.txt abc.txt", func(t *testing.T) { + if err := fs.Rename("/abc/abc.txt", "/abc.txt"); err != nil { + t.Fatalf("Failed to move abc.txt to root: %v", err) + } + + stat, err := fs.Stat("/abc.txt") + if err != nil { + t.Fatalf("Failed to stat moved file: %v", err) + } + if stat.IsDir() { + t.Error("abc.txt should be a file") + } + if stat.Size() != 100 { + t.Errorf("File size should remain 100, got %d", stat.Size()) + } + + if _, err := fs.Stat("/abc/abc.txt"); err == nil { + t.Error("File should not exist in old location") + } + }) +} diff --git a/internal/errors.go b/internal/errors.go new file mode 100644 index 0000000..0dd802a --- /dev/null +++ b/internal/errors.go @@ -0,0 +1,18 @@ +package internal + +import ( + "errors" + "os" +) + +var ( + ErrIsDir = &os.PathError{Err: errors.New("is a directory")} + ErrIsNotDir = &os.PathError{Err: errors.New("is not a directory")} + ErrNotFound = &os.PathError{Err: errors.New("source or destination does not exist")} + ErrNotEmpty = &os.PathError{Err: errors.New("directory not empty")} + ErrInvalidSeek = &os.PathError{Err: errors.New("invalid seek offset")} + ErrNotSupported = &os.PathError{Err: errors.New("fs doesn't support this operation")} + ErrAlreadyExist = &os.PathError{Err: errors.New("destination already exist")} + ErrInvalidOperation = &os.PathError{Err: errors.New("invalid operation - hint: trying to move directory into itself")} + ErrInvalidRootOperation = &os.PathError{Err: errors.New("invalid operation - stop fucking with root directory")} +) diff --git a/internal/filesystem/file.go b/internal/filesystem/file.go new file mode 100644 index 0000000..43a50a2 --- /dev/null +++ b/internal/filesystem/file.go @@ -0,0 +1,208 @@ +package filesystem + +import ( + "io" + "os" + + "fafda/internal" +) + +type File struct { + *internal.Node + + flag int + + off int64 + dirCount int + written int64 + writer io.WriteCloser + reader io.ReadCloser + + driver internal.StorageDriver + meta internal.MetaFileSystem +} + +func NewFile( + flag int, + node *internal.Node, + metafs internal.MetaFileSystem, + driver internal.StorageDriver, +) *File { + return &File{ + Node: node, + flag: flag, + + off: 0, + dirCount: 0, + written: 0, + writer: nil, + reader: nil, + + driver: driver, + meta: metafs, + } +} + +func (f *File) Truncate(_ int64) error { return internal.ErrNotSupported } +func (f *File) WriteAt(_ []byte, _ int64) (int, error) { return 0, internal.ErrNotSupported } +func (f *File) Sync() error { return nil } + +func (f *File) Readdirnames(n int) ([]string, error) { + if !f.IsDir() { + return nil, internal.ErrIsNotDir + } + fi, err := f.Readdir(n) + names := make([]string, len(fi)) + for i, f := range fi { + names[i] = f.Name() + } + + return names, err +} + +func (f *File) Readdir(n int) ([]os.FileInfo, error) { + if !f.IsDir() { + return nil, internal.ErrIsNotDir + } + + // If n > 0, return at most n entries + // If n <= 0, return all remaining entries + files, err := f.meta.Ls(f.Path(), n, f.dirCount) + if err != nil { + return nil, err + } + + entries := make([]os.FileInfo, len(files)) + for i, file := range files { + entries[i] = &file + } + + f.dirCount += len(entries) + + if n > 0 && len(entries) == 0 { + return entries, io.EOF + } + + return entries, err +} + +func (f *File) Read(p []byte) (n int, err error) { + if f.IsDir() { + return 0, internal.ErrIsDir + } + if f.reader == nil { + if err = f.openReadStream(0); err != nil { + return 0, err + } + } + n, err = f.reader.Read(p) + // Do not increment n on failed read + if err != nil && err != io.EOF { + return n, err + } + f.off += int64(n) + return n, err +} + +func (f *File) ReadAt(p []byte, off int64) (n int, err error) { + if f.IsDir() { + return 0, internal.ErrIsDir + } + if _, err := f.Seek(off, io.SeekCurrent); err != nil { + return 0, err + } + return f.Read(p) +} + +func (f *File) WriteString(s string) (ret int, err error) { + if f.IsDir() { + return 0, internal.ErrIsDir + } + return f.Write([]byte(s)) +} + +func (f *File) Write(p []byte) (int, error) { + if f.IsDir() { + return 0, internal.ErrIsDir + } + + if !checkFlags(os.O_WRONLY, f.flag) { + return 0, internal.ErrNotSupported + } + + if f.writer == nil { + f.written = 0 + if err := f.driver.Truncate(f.Id()); err != nil { + return 0, err + } + if writer, err := f.driver.GetWriter(f.Id()); err != nil { + return 0, err + } else { + f.writer = writer + } + } + n, err := f.writer.Write(p) + f.written += int64(n) + + return n, err +} + +func (f *File) Seek(offset int64, whence int) (int64, error) { + if f.IsDir() { + return 0, internal.ErrIsDir + } + + pos := int64(0) + + switch whence { + case io.SeekStart: + pos = offset + case io.SeekCurrent: + pos = f.off + offset + case io.SeekEnd: + pos = f.Size() - offset + } + if pos < 0 { + return 0, internal.ErrInvalidSeek + } + if f.reader != nil { + if err := f.reader.Close(); err != nil { + return 0, err + } + } + f.reader = nil + if err := f.openReadStream(pos); err != nil { + return 0, err + } + + return pos, nil +} + +func (f *File) Close() error { + if f.writer != nil { + if err := f.writer.Close(); err != nil { + return err + } + if err := f.meta.Sync(f.Path(), f.written); err != nil { + return err + } + f.writer = nil + } + if f.reader != nil { + if err := f.reader.Close(); err != nil { + return err + } + f.reader = nil + } + + return nil +} + +func (f *File) openReadStream(startAt int64) error { + if reader, err := f.driver.GetReader(f.Id(), startAt); err != nil { + return err + } else { + f.reader = reader + } + return nil +} diff --git a/internal/filesystem/filesystem.go b/internal/filesystem/filesystem.go new file mode 100644 index 0000000..4ba4b71 --- /dev/null +++ b/internal/filesystem/filesystem.go @@ -0,0 +1,83 @@ +package filesystem + +import ( + "errors" + "fmt" + "os" + "time" + + "github.com/spf13/afero" + + "fafda/internal" + "fafda/pkg" +) + +type Fs struct { + meta internal.MetaFileSystem + driver internal.StorageDriver +} + +func New(driver internal.StorageDriver, dp internal.MetaFileSystem) afero.Fs { + return pkg.NewLogFS(&Fs{driver: driver, meta: dp}) +} + +func (fs *Fs) Name() string { return "WhyAreYouGayFs" } +func (fs *Fs) Chown(_ string, _, _ int) error { return internal.ErrNotSupported } +func (fs *Fs) Chmod(_ string, _ os.FileMode) error { return internal.ErrNotSupported } +func (fs *Fs) Remove(path string) error { return fs.meta.Remove(path) } +func (fs *Fs) RemoveAll(path string) error { return fs.meta.RemoveAll(path) } +func (fs *Fs) Rename(oldname, newname string) error { return fs.meta.Rename(oldname, newname) } +func (fs *Fs) Stat(path string) (os.FileInfo, error) { return fs.meta.Stat(path) } +func (fs *Fs) Chtimes(path string, _, mtime time.Time) error { return fs.meta.Chtimes(path, mtime) } +func (fs *Fs) Mkdir(path string, _ os.FileMode) error { return fs.meta.Mkdir(path) } +func (fs *Fs) MkdirAll(path string, _ os.FileMode) error { return fs.meta.MkdirAll(path) } + +func (fs *Fs) Create(path string) (afero.File, error) { + if err := fs.meta.Touch(path); err != nil { + return nil, err + } + return fs.OpenFile(path, os.O_WRONLY, 0666) +} + +func (fs *Fs) Open(path string) (afero.File, error) { + f, err := fs.meta.Stat(path) + if err != nil { + return nil, err + } + file := NewFile(os.O_RDONLY, f, fs.meta, fs.driver) + + return file, nil +} + +func (fs *Fs) OpenFile(name string, flag int, _ os.FileMode) (afero.File, error) { + allowedFlags := os.O_WRONLY | os.O_RDONLY | os.O_CREATE | os.O_TRUNC + + if !checkFlags(flag, allowedFlags) { + return nil, fmt.Errorf("flag not supported") + } + + f, err := fs.meta.Stat(name) + if err != nil { + if errors.Is(err, internal.ErrNotFound) && checkFlags(os.O_CREATE, flag) { + return fs.Create(name) + } + return nil, err + } + + if checkFlags(os.O_TRUNC, flag) { + if err = fs.driver.Truncate(f.Id()); err != nil { + return nil, err + } + if err = fs.meta.Sync(f.Path(), 0); err != nil { + return nil, err + } + } + + file := NewFile(flag, f, fs.meta, fs.driver) + + return file, nil +} + +func checkFlags(flag int, allowedFlags int) bool { + return flag == (flag & allowedFlags) +} diff --git a/internal/ftp/server.go b/internal/ftp/server.go new file mode 100644 index 0000000..627316b --- /dev/null +++ b/internal/ftp/server.go @@ -0,0 +1,128 @@ +package ftp + +import ( + "crypto/tls" + "errors" + "fmt" + "io" + "net/http" + + "github.com/fclairamb/ftpserverlib" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/afero" + + "fafda/config" +) + +const IPResolveURL = "https://ipinfo.io/ip" + +var ( + ErrNoTLS = errors.New("TLS is not configured") + ErrBadUserNameOrPassword = errors.New("bad username or password") +) + +func Serv(cfg config.FTPServer, fs afero.Fs) error { + logger := log.With().Str("component", "ftpserver").Logger() + + driver := &Driver{ + Fs: fs, + Debug: true, + username: cfg.Username, + password: cfg.Password, + Settings: &ftpserver.Settings{ + ListenAddr: cfg.Addr, + DefaultTransferType: ftpserver.TransferTypeBinary, + // Stooopid FTP thinks connection is idle, even when file transfer is going on. + // Default is 900 seconds after which the server will drop the connection + // Increased it to 24 hours to allow big file transfers + IdleTimeout: 86400, // 24 hour + }, + logger: logger, + } + + if cfg.PortRange != nil { + portRange := &ftpserver.PortRange{} + + if cfg.PortRange.Start < 1 || cfg.PortRange.Start > 65535 { + return fmt.Errorf("invalid start port: must be between 1-65535, got %d", cfg.PortRange.Start) + } + if cfg.PortRange.End < 1 || cfg.PortRange.End > 65535 { + return fmt.Errorf("invalid end port: must be between 1-65535, got %d", cfg.PortRange.End) + } + + if cfg.PortRange.Start >= cfg.PortRange.End { + return fmt.Errorf("start port (%d) must be less than end port (%d)", + cfg.PortRange.Start, cfg.PortRange.End) + } + + portRange.Start = cfg.PortRange.Start + portRange.End = cfg.PortRange.End + + driver.Settings.PassiveTransferPortRange = portRange + driver.Settings.PublicIPResolver = func(context ftpserver.ClientContext) (string, error) { + resp, err := http.Get(IPResolveURL) + if err != nil { + return "", err + } + ip, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(ip), nil + } + } + + server := ftpserver.NewFtpServer(driver) + logger.Info().Str("address", cfg.Addr).Msg("starting server") + + return server.ListenAndServe() +} + +type Driver struct { + Fs afero.Fs + Debug bool + Settings *ftpserver.Settings + username string + password string + logger zerolog.Logger +} + +func (d *Driver) ClientConnected(cc ftpserver.ClientContext) (string, error) { + d.logger.Info(). + Str("address", cc.RemoteAddr().String()). + Str("version", cc.GetClientVersion()). + Uint32("sessionId", cc.ID()). + Msg("client connected") + return "Fafda FTP Server", nil +} + +func (d *Driver) ClientDisconnected(cc ftpserver.ClientContext) { + d.logger.Info(). + Str("address", cc.RemoteAddr().String()). + Str("version", cc.GetClientVersion()). + Uint32("sessionId", cc.ID()). + Msg("client disconnected") +} + +func (d *Driver) AuthUser(cc ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) { + if d.username != "" && d.username != user || d.password != "" && d.password != pass { + d.logger.Warn(). + Str("address", cc.RemoteAddr().String()). + Uint32("session_id", cc.ID()). + Str("user", user). + Err(ErrBadUserNameOrPassword). + Msg("authentication failed") + return nil, ErrBadUserNameOrPassword + } + d.logger.Info(). + Str("address", cc.RemoteAddr().String()). + Uint32("sessionId", cc.ID()). + Str("user", user). + Msg("authentication successful") + return d.Fs, nil +} + +func (d *Driver) GetSettings() (*ftpserver.Settings, error) { return d.Settings, nil } + +func (d *Driver) GetTLSConfig() (*tls.Config, error) { return nil, ErrNoTLS } diff --git a/internal/github/asset.go b/internal/github/asset.go new file mode 100644 index 0000000..edb4f8e --- /dev/null +++ b/internal/github/asset.go @@ -0,0 +1,133 @@ +package github + +import ( + "bytes" + "encoding/gob" + "fmt" + "io" + "sort" + + "go.etcd.io/bbolt" +) + +type Asset struct { + Id int + Name string + Username string + Repository string + ReleaseId int + ReleaseTag string + Size int + Number int + + client *Client +} + +func (a *Asset) GetSize() int { + return a.Size +} + +func (a *Asset) GetReader(start, end int) (io.ReadCloser, error) { + return a.client.DownloadAsset(a, start, end) +} + +func (a *Asset) url() string { + return fmt.Sprintf( + "%s/repos/%s/%s/releases/assets/%d", + apiURL, a.Username, a.Repository, a.Id, + ) +} + +func (a *Asset) publicURL() string { + return fmt.Sprintf( + "https://github.com/%s/%s/releases/download/%s/%s", + a.Username, a.Repository, a.ReleaseTag, a.Name, + ) +} + +type AssetStore struct { + db *bbolt.DB + bucketName []byte +} + +func NewAssetStore(db *bbolt.DB) (*AssetStore, error) { + + err := db.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucketIfNotExists(assetBucket) + if err != nil { + return fmt.Errorf("failed to create file bucket %w", err) + } + return nil + }) + if err != nil { + return nil, err + } + + return &AssetStore{ + db: db, + bucketName: assetBucket, + }, nil +} + +func (ass *AssetStore) Write(fileId string, assets []*Asset) error { + return ass.db.Update(func(tx *bbolt.Tx) error { + bucket, err := tx.CreateBucketIfNotExists(ass.bucketName) + if err != nil { + return err + } + + var buf bytes.Buffer + if err := gob.NewEncoder(&buf).Encode(assets); err != nil { + return err + } + + key := []byte(fileId) + return bucket.Put(key, buf.Bytes()) + }) +} + +func (ass *AssetStore) Get(fileId string) ([]Asset, error) { + var assets []Asset + + err := ass.db.View(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(ass.bucketName) + if bucket == nil { + return nil + } + + data := bucket.Get([]byte(fileId)) + if data == nil { + return nil + } + + buf := bytes.NewBuffer(data) + return gob.NewDecoder(buf).Decode(&assets) + }) + sort.Slice(assets, func(i, j int) bool { + return assets[i].Number < assets[j].Number + }) + return assets, err +} + +func (ass *AssetStore) Size(fileId string) (int64, error) { + assets, err := ass.Get(fileId) + if err != nil { + return 0, err + } + size := int64(0) + for _, asset := range assets { + size += int64(asset.Size) + } + return size, err +} + +func (ass *AssetStore) Delete(fileId string) error { + return ass.db.Update(func(tx *bbolt.Tx) error { + bucket := tx.Bucket(ass.bucketName) + if bucket == nil { + return nil + } + + return bucket.Delete([]byte(fileId)) + }) +} diff --git a/internal/github/assetnames.go b/internal/github/assetnames.go new file mode 100644 index 0000000..937f62f --- /dev/null +++ b/internal/github/assetnames.go @@ -0,0 +1,236 @@ +package github + +import ( + "math/rand" + + nanoid "github.com/matoous/go-nanoid/v2" +) + +var commonAssetNames = []string{ + "app-darwin-x64.dmg", + "app-linux-x64.AppImage", + "app-mac.zip", + "app-setup-x64.exe", + "app-win32-x64.exe", + "app-win64.msi", + "app.exe", + "app.pkg", + "app_amd64.deb", + "app_arm64.deb", + + "archive.tar.gz", + "binaries.zip", + "bundle.tar.xz", + "dist.zip", + "package.tar.bz2", + "release.7z", + "source.zip", + "build.zip", + "artifacts.tar", + "compressed.rar", + + "darwin-amd64", + "linux-aarch64", + "linux-arm64", + "linux-armv7", + "linux-x86_64", + "macos-universal", + "windows-386", + "windows-amd64", + "x86_64-unknown-linux-gnu", + "x86_64-apple-darwin", + + "app-release.apk", + "app-debug.apk", + "app.ipa", + "android-release.aab", + "android-debug.apk", + "ios-release.ipa", + "mobile-release.apk", + "mobile-debug.apk", + "release-unsigned.apk", + "signed-release.apk", + + "lib.so", + "lib.dll", + "lib.dylib", + "module.jar", + "package.gem", + "plugin.so", + "runtime.dll", + "shared.so", + "static.lib", + "framework.dll", + + "docs.pdf", + "manual.pdf", + "reference.html", + "changelog.md", + "readme.txt", + "guide.pdf", + "documentation.zip", + "assets.zip", + "resources.tar.gz", + "media.zip", + + "debug-symbols.pdb", + "source-maps.zip", + "headers.zip", + "include.zip", + "dev-tools.zip", + "sdk.zip", + "api-docs.zip", + "examples.zip", + "samples.zip", + "templates.zip", + + "container.tar", + "docker-image.tar", + "vm-disk.vmdk", + "virtual-appliance.ova", + "image.iso", + "disk-image.raw", + "snapshot.qcow2", + "backup.vdi", + "system-image.img", + "bootable.iso", + + "config.json", + "settings.yaml", + "data.db", + "database.sql", + "preferences.xml", + "metadata.json", + "schema.sql", + "rules.yaml", + "translations.json", + "locales.zip", + + "extension.vsix", + "plugin.jar", + "addon.xpi", + "module.npm", + "package.pip", + "component.crx", + "widget.wgt", + "theme.zip", + "skin.zip", + "mod.pak", + + "app-universal-binary", + "app-cross-platform.zip", + "multi-arch-bundle.tar.gz", + "all-platforms.zip", + "unified-release.zip", + "portable-version.zip", + "standalone-release.zip", + "cross-compile-bundle.tar", + "hybrid-package.zip", + "platform-agnostic.zip", + + "python-wheel.whl", + "node-module.tgz", + "ruby-gem.gem", + "java-bundle.jar", + "perl-module.tar.gz", + "php-package.phar", + "rust-crate.crate", + "golang-pkg.tar.gz", + "dotnet-nuget.nupkg", + "swift-package.swiftpm", + + "frontend-bundle.js", + "styles.min.css", + "web-assets.zip", + "static-files.tar.gz", + "website-bundle.zip", + "spa-release.zip", + "cdn-assets.zip", + "web-components.zip", + "client-bundle.js", + "webapp-dist.zip", + + "test-suite.zip", + "debug-build.zip", + "testing-tools.tar.gz", + "benchmark-suite.zip", + "profiler-tools.zip", + "coverage-report.zip", + "performance-tests.zip", + "unit-tests.zip", + "integration-tests.zip", + "stress-test-suite.zip", + + "checksums.txt", + "signatures.asc", + "pgp-keys.asc", + "hash-verification.md", + "security-bundle.zip", + "encrypted-assets.zip", + "certificate-bundle.zip", + "key-store.jks", + "trust-chain.pem", + "signature-files.zip", + + "lambda-function.zip", + "server-bundle.tar.gz", + "cloud-function.zip", + "deployment-package.zip", + "serverless-bundle.zip", + "microservice-build.zip", + "cluster-config.tar.gz", + "node-service.zip", + "container-bundle.tar", + "scaling-scripts.zip", + + "game-assets.pak", + "textures.zip", + "models-3d.zip", + "sound-effects.zip", + "level-data.bin", + "animation-sets.zip", + "shader-pack.zip", + "particle-effects.zip", + "game-maps.zip", + "cutscenes.tar.gz", + + "model-weights.h5", + "trained-model.pkl", + "neural-network.pt", + "dataset-bundle.zip", + "feature-vectors.npz", + "tensorflow-model.pb", + "ml-pipeline.zip", + "training-data.tar.gz", + "inference-model.onnx", + "embeddings.bin", + + "firmware.bin", + "device-drivers.zip", + "hardware-specs.pdf", + "schematic-files.zip", + "pcb-designs.zip", + "protocol-definitions.xml", + "sensor-configs.json", + "device-profiles.yaml", + "bootloader.img", + "embedded-system.hex", + + "linter-rules.zip", + "formatter-config.zip", + "code-snippets.zip", + "ide-plugins.zip", + "build-tools.tar.gz", + "compiler-extensions.zip", + "analysis-tools.zip", + "debugging-symbols.zip", + "profiling-tools.tar.gz", + "development-kit.zip", +} + +func getRandomAssetName() string { + randomIndex := int(rand.Int63n(int64(len(commonAssetNames)))) + assetName := commonAssetNames[randomIndex] + + return nanoid.Must() + assetName +} diff --git a/internal/github/client.go b/internal/github/client.go new file mode 100644 index 0000000..ff2f15b --- /dev/null +++ b/internal/github/client.go @@ -0,0 +1,142 @@ +package github + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "fafda/config" + "fafda/internal" +) + +const headerRateLimitRetryAfter = "retry-after" +const headerRateLimitReset = "x-ratelimit-remaining" +const headerRateLimitRemaining = "x-ratelimit-reset" + +type Client struct { + partSize int + client *http.Client + resources *ReleaseManager +} + +func NewClient(cfg config.GitHub) (*Client, error) { + resources, err := NewReleaseManager(cfg) + if err != nil { + return nil, fmt.Errorf("failed to initialize resource manager: %w", err) + } + + return &Client{ + client: &http.Client{}, + resources: resources, + }, nil +} + +func handleRateLimit(resp *http.Response) time.Duration { + if resp.StatusCode != http.StatusForbidden && + resp.StatusCode != http.StatusTooManyRequests { + return 0 + } + + if retryAfter := resp.Header.Get(headerRateLimitRetryAfter); retryAfter != "" { + seconds, _ := strconv.ParseInt(retryAfter, 10, 64) + return time.Duration(seconds) * time.Second + } + + if resp.Header.Get(headerRateLimitRemaining) == "0" { + resetTime, _ := strconv.ParseInt(resp.Header.Get(headerRateLimitReset), 10, 64) + waitTime := resetTime - time.Now().UTC().Unix() + if waitTime > 0 { + return time.Duration(waitTime) * time.Second + } + } + + // Should never come to this - FUCK YOU GITHUB + return time.Minute +} + +func (c *Client) doRequest(req *http.Request) (*http.Response, error) { + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + + if waitTime := handleRateLimit(resp); waitTime > 0 { + time.Sleep(waitTime) + return c.doRequest(req) + } + + return resp, nil +} + +func (c *Client) UploadAsset(filename string, size int64, b []byte) (*Asset, error) { + release := c.resources.GetNextRelease() + url := fmt.Sprintf( + "%s/repos/%s/%s/releases/%d/assets", + uploadURL, release.Username, release.Repository, release.ReleaseId, + ) + + req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s?name=%s", url, filename), bytes.NewReader(b)) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + req.Header.Set(internal.HeaderAccept, internal.MediaTypeGithubJSON) + req.Header.Set(internal.HeaderContentType, internal.MediaTypeOctetStream) + req.Header.Set(internal.HeaderAuthorization, "Bearer "+release.AuthToken) + req.ContentLength = size + + resp, err := c.doRequest(req) + if err != nil { + return nil, fmt.Errorf("upload asset: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("upload asset failed: %s", string(body)) + } + + var asset Asset + if err := json.NewDecoder(resp.Body).Decode(&asset); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + + asset.Name = filename + asset.Username = release.Username + asset.Repository = release.Repository + return &asset, nil +} + +func (c *Client) DownloadAsset(asset *Asset, start, end int) (io.ReadCloser, error) { + token := c.resources.GetUserToken(asset.Username) + + if token == "" { + return nil, fmt.Errorf("token not found for given asset username:%s", asset.Username) + } + + req, err := http.NewRequest(http.MethodGet, asset.url(), nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + req.Header.Set(internal.HeaderAuthorization, "Bearer "+token) + req.Header.Set(internal.HeaderAccept, internal.MediaTypeOctetStream) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + + resp, err := c.doRequest(req) + if err != nil { + return nil, fmt.Errorf("download asset: %w", err) + } + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + return nil, fmt.Errorf("download asset failed: %s", string(body)) + } + + return resp.Body, nil +} diff --git a/internal/github/consts.go b/internal/github/consts.go new file mode 100644 index 0000000..d064971 --- /dev/null +++ b/internal/github/consts.go @@ -0,0 +1,5 @@ +package github + +var assetBucket = []byte("assets") +var apiURL = "https://api.github.com" +var uploadURL = "https://uploads.github.com" diff --git a/internal/github/driver.go b/internal/github/driver.go new file mode 100644 index 0000000..9b029ba --- /dev/null +++ b/internal/github/driver.go @@ -0,0 +1,59 @@ +package github + +import ( + "fmt" + "io" + + "go.etcd.io/bbolt" + + "fafda/config" +) + +const MaxPartSize = (2 * 1024 * 1024 * 1024) - 429496729 // 2GB - 20% + +type Driver struct { + client *Client + ass *AssetStore + + partSize int64 + concurrency int +} + +func NewDriver(cfg config.GitHub, db *bbolt.DB) (*Driver, error) { + + if cfg.PartSize <= 0 || cfg.PartSize > MaxPartSize { + return nil, fmt.Errorf("partSize must be positive and under ") + } + + client, err := NewClient(cfg) + if err != nil { + return nil, err + } + ass, err := NewAssetStore(db) + if err != nil { + return nil, err + } + + return &Driver{ + ass: ass, + client: client, + partSize: cfg.PartSize, + concurrency: cfg.Concurrency, + }, nil +} + +func (d *Driver) GetReader(fileId string, pos int64) (io.ReadCloser, error) { + return NewReader(fileId, pos, d) +} + +func (d *Driver) GetWriter(fileId string) (io.WriteCloser, error) { + return NewWriter(fileId, d) +} + +func (d *Driver) GetSize(fileId string) (int64, error) { + return d.ass.Size(fileId) +} + +func (d *Driver) Truncate(fileId string) error { + return d.ass.Delete(fileId) +} diff --git a/internal/github/lsreleases.go b/internal/github/lsreleases.go new file mode 100644 index 0000000..b5ff4a4 --- /dev/null +++ b/internal/github/lsreleases.go @@ -0,0 +1,135 @@ +package github + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" +) + +type Repository struct { + Name string `json:"name"` + FullName string `json:"full_name"` + Owner struct { + Login string `json:"login"` + } `json:"owner"` +} + +type Release struct { + Id int64 `json:"id"` + TagName string `json:"tag_name"` +} + +type ReleaseInfo struct { + AuthToken string `json:"authToken"` + Username string `json:"username"` + Repository string `json:"repository"` + ReleaseId int64 `json:"releaseId"` + ReleaseTag string `json:"releaseTag"` +} + +func fetchGitHubAPI(token, url string) ([]byte, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) + } + + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Accept", "application/vnd.github.v3+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("error making request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("API request failed with status: %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response: %v", err) + } + + return body, nil +} + +func getRepositories(token string) ([]Repository, error) { + body, err := fetchGitHubAPI(token, "https://api.github.com/user/repos?per_page=100") + if err != nil { + return nil, err + } + + var repos []Repository + if err := json.Unmarshal(body, &repos); err != nil { + return nil, fmt.Errorf("error parsing repositories: %v", err) + } + + return repos, nil +} + +func getReleases(token, repoFullName string) ([]Release, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/releases", repoFullName) + body, err := fetchGitHubAPI(token, url) + if err != nil { + return nil, err + } + + var releases []Release + if err := json.Unmarshal(body, &releases); err != nil { + return nil, fmt.Errorf("error parsing releases: %v", err) + } + + return releases, nil +} + +func GetAllReleasesInfo(tokens []string) ([]ReleaseInfo, error) { + var allReleases []ReleaseInfo + + for _, token := range tokens { + repos, err := getRepositories(token) + if err != nil { + fmt.Printf("Warning: error fetching repositories for token: %v\n", err) + continue + } + + for _, repo := range repos { + releases, err := getReleases(token, repo.FullName) + if err != nil { + fmt.Printf("Warning: error fetching releases for %s: %v\n", repo.FullName, err) + continue + } + + for _, release := range releases { + releaseInfo := ReleaseInfo{ + Username: strings.ToLower(repo.Owner.Login), + Repository: repo.Name, + ReleaseId: release.Id, + ReleaseTag: release.TagName, + AuthToken: token, + } + allReleases = append(allReleases, releaseInfo) + } + } + } + + return allReleases, nil +} + +func ListReleases(tokens []string) { + releases, err := GetAllReleasesInfo(tokens) + if err != nil { + fmt.Printf("Error: %v\n", err) + return + } + + output, err := json.MarshalIndent(releases, "", " ") + if err != nil { + fmt.Printf("Error marshaling to JSON: %v\n", err) + return + } + + fmt.Println(string(output)) +} diff --git a/internal/github/reader.go b/internal/github/reader.go new file mode 100644 index 0000000..cbda54c --- /dev/null +++ b/internal/github/reader.go @@ -0,0 +1,41 @@ +package github + +import ( + "fmt" + "io" + + "fafda/internal/partedio" +) + +type Reader struct { + drvr *Driver + reader io.ReadCloser +} + +func NewReader(fileId string, pos int64, drvr *Driver) (*Reader, error) { + assets, err := drvr.ass.Get(fileId) + if err != nil { + return nil, err + } + if len(assets) == 0 { + return nil, fmt.Errorf("assets len is zero") + } + partReaders := make([]partedio.PartReader, len(assets)) + for i, asset := range assets { + asset.client = drvr.client + partReaders[i] = &asset + } + reader, err := partedio.NewReader(partReaders, pos) + if err != nil { + return nil, err + } + return &Reader{reader: reader}, nil +} + +func (r *Reader) Read(p []byte) (int, error) { + return r.reader.Read(p) +} + +func (r *Reader) Close() error { + return r.reader.Close() +} diff --git a/internal/github/releasemanager.go b/internal/github/releasemanager.go new file mode 100644 index 0000000..1ade934 --- /dev/null +++ b/internal/github/releasemanager.go @@ -0,0 +1,55 @@ +package github + +import ( + "fmt" + "sync" + + "fafda/config" +) + +type ReleaseManager struct { + releases []config.GitHubRelease + userTokens map[string]string + currentToken int + currentRel int + currentPair int + mu sync.Mutex +} + +func NewReleaseManager(cfg config.GitHub) (*ReleaseManager, error) { + rm := &ReleaseManager{ + userTokens: map[string]string{}, + releases: make([]config.GitHubRelease, 0), + currentToken: -1, + currentRel: -1, + currentPair: -1, + } + + for _, release := range cfg.Releases { + if release.AuthToken == "" { + return nil, fmt.Errorf("auth token missing for release %d", release.ReleaseId) + } + rm.userTokens[release.Username] = release.AuthToken + if !release.ReadOnly { + rm.releases = append(rm.releases, release) + } + } + + if len(rm.releases) == 0 { + return nil, fmt.Errorf("no valid writable release found in config") + } + + return rm, nil +} + +func (rm *ReleaseManager) GetNextRelease() config.GitHubRelease { + rm.mu.Lock() + defer rm.mu.Unlock() + + rm.currentPair = (rm.currentPair + 1) % len(rm.releases) + return rm.releases[rm.currentPair] +} + +func (rm *ReleaseManager) GetUserToken(user string) string { + return rm.userTokens[user] +} diff --git a/internal/github/writer.go b/internal/github/writer.go new file mode 100644 index 0000000..1ae3c4d --- /dev/null +++ b/internal/github/writer.go @@ -0,0 +1,65 @@ +package github + +import ( + "io" + "math/rand" + + "fafda/internal/partedio" +) + +type Writer struct { + fileId string + drvr *Driver + writer io.WriteCloser + assets []*Asset +} + +func NewWriter(fileId string, drvr *Driver) (*Writer, error) { + writer := &Writer{ + fileId: fileId, + drvr: drvr, + assets: make([]*Asset, 0), + } + + partSize := randomPartSize(drvr.partSize, 20) + w, err := partedio.NewNWriter(partSize, drvr.concurrency, writer.processor) + if err != nil { + return nil, err + } + + writer.writer = w + return writer, nil +} + +func (w *Writer) Assets() []*Asset { + return w.assets +} + +func (w *Writer) processor(partNum int, partSize int64, data []byte) error { + assetName := getRandomAssetName() + asset, err := w.drvr.client.UploadAsset(assetName, partSize, data) + if err != nil { + return err + } + asset.Number = partNum + w.assets = append(w.assets, asset) + return nil +} + +func (w *Writer) Write(p []byte) (int, error) { + return w.writer.Write(p) +} + +func (w *Writer) Close() error { + if err := w.writer.Close(); err != nil { + return err + } + return w.drvr.ass.Write(w.fileId, w.assets) +} + +func randomPartSize(baseNumber int64, percentageRange int) int64 { + minValue := float64(baseNumber) * (1 - float64(percentageRange)/100) + maxValue := float64(baseNumber) * (1 + float64(percentageRange)/100) + + return int64(minValue + rand.Float64()*(maxValue-minValue)) +} diff --git a/internal/http.go b/internal/http.go new file mode 100644 index 0000000..7f189ba --- /dev/null +++ b/internal/http.go @@ -0,0 +1,14 @@ +package internal + +const ( + HeaderAccept = "Accept" + HeaderContentType = "Content-Type" + HeaderContentLength = "Content-Length" + HeaderAuthorization = "Authorization" +) + +const ( + MediaTypeJOSN = "application/json" + MediaTypeGithubJSON = "application/vnd.github+json" + MediaTypeOctetStream = "application/octet-stream" +) diff --git a/internal/http/http.go b/internal/http/http.go new file mode 100644 index 0000000..78a9bc7 --- /dev/null +++ b/internal/http/http.go @@ -0,0 +1,21 @@ +package http + +import ( + "net/http" + + "github.com/rs/zerolog/log" + "github.com/spf13/afero" + + "fafda/config" +) + +func Serv(cfg config.HTTPServer, fs afero.Fs) error { + httpFs := afero.NewHttpFs(fs) + fileServer := http.FileServer(httpFs.Dir("/")) + http.Handle("/", fileServer) + log.Info(). + Str("component", "httpserver"). + Str("address", cfg.Addr). + Msg("starting server") + return http.ListenAndServe(cfg.Addr, nil) +} diff --git a/internal/node.go b/internal/node.go new file mode 100644 index 0000000..c92ec96 --- /dev/null +++ b/internal/node.go @@ -0,0 +1,84 @@ +package internal + +import ( + "bytes" + "encoding/gob" + "os" + "path" + "time" +) + +type Node struct { + id string + path string + name string + isDir bool + size int64 + mode os.FileMode + createdAt time.Time + modTime time.Time +} + +func (n *Node) Id() string { return n.id } +func (n *Node) Name() string { return path.Base(n.path) } +func (n *Node) Size() int64 { return n.size } +func (n *Node) Mode() os.FileMode { return n.mode } +func (n *Node) ModTime() time.Time { return n.modTime } +func (n *Node) IsDir() bool { return n.isDir } +func (n *Node) Sys() interface{} { return nil } +func (n *Node) Stat() (os.FileInfo, error) { return n, nil } +func (n *Node) Path() string { return n.path } + +func (n *Node) SetId(id string) *Node { n.id = id; return n } +func (n *Node) SetPath(path string) *Node { n.path = path; return n } +func (n *Node) SetIsDir(isDir bool) *Node { n.isDir = isDir; return n } +func (n *Node) SetSize(size int64) *Node { n.size = size; return n } +func (n *Node) SetMode(mode os.FileMode) *Node { n.mode = mode; return n } +func (n *Node) SetCreatedAt(t time.Time) *Node { n.createdAt = t; return n } +func (n *Node) SetModTime(t time.Time) *Node { n.modTime = t; return n } + +type nodeAlias struct { + Id string + Path string + Name string + IsDir bool + Size int64 + Mode os.FileMode + CreatedAt time.Time + ModTime time.Time +} + +func (n *Node) GobEncode() ([]byte, error) { + var buf bytes.Buffer + if err := gob.NewEncoder(&buf).Encode(nodeAlias{ + Id: n.id, + Path: n.path, + Name: n.name, + IsDir: n.isDir, + Size: n.size, + Mode: n.mode, + CreatedAt: n.createdAt, + ModTime: n.modTime, + }); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (n *Node) GobDecode(data []byte) error { + var alias nodeAlias + if err := gob.NewDecoder(bytes.NewBuffer(data)).Decode(&alias); err != nil { + return err + } + + n.id = alias.Id + n.path = alias.Path + n.name = alias.Name + n.isDir = alias.IsDir + n.size = alias.Size + n.mode = alias.Mode + n.createdAt = alias.CreatedAt + n.modTime = alias.ModTime + + return nil +} diff --git a/internal/partedio/errors.go b/internal/partedio/errors.go new file mode 100644 index 0000000..6221243 --- /dev/null +++ b/internal/partedio/errors.go @@ -0,0 +1,10 @@ +package partedio + +import ( + "errors" +) + +var ( + ErrClosed = errors.New("is closed") + ErrNoParts = errors.New("no parts provided") +) diff --git a/internal/partedio/nwriter.go b/internal/partedio/nwriter.go new file mode 100644 index 0000000..57971bc --- /dev/null +++ b/internal/partedio/nwriter.go @@ -0,0 +1,126 @@ +package partedio + +import ( + "fmt" + "io" + "sync" + "sync/atomic" +) + +type PartHandler func(partNum int, contentLength int64, data []byte) error + +var ( + pool *sync.Pool + initOnce sync.Once +) + +type NWriter struct { + partSize int64 + concurrency int + handler PartHandler + closed bool + pwriter *io.PipeWriter + partCount int64 + err error + + mu sync.Mutex + wg sync.WaitGroup +} + +func NewNWriter(partSize int64, concurrency int, handler PartHandler) (io.WriteCloser, error) { + if partSize <= 0 || concurrency <= 0 { + return nil, fmt.Errorf("part size and concurrency must be positive") + } + if handler == nil { + return nil, fmt.Errorf("handler function cannot be nil") + } + + initOnce.Do(func() { + pool = &sync.Pool{ + New: func() interface{} { + return make([]byte, partSize) + }, + } + }) + + reader, writer := io.Pipe() + w := &NWriter{ + handler: handler, + partSize: partSize, + pwriter: writer, + concurrency: concurrency, + } + go w.startWriting(NewSyncReader(reader)) + + return w, nil +} + +func (nw *NWriter) Write(p []byte) (int, error) { + if nw.closed { + return 0, ErrClosed + } + if nw.err != nil { + return 0, nw.err + } + return nw.pwriter.Write(p) +} + +func (nw *NWriter) Close() error { + if nw.closed { + return ErrClosed + } + nw.closed = true + if nw.pwriter != nil { + if err := nw.pwriter.Close(); err != nil { + return err + } + } + nw.wg.Wait() + return nw.getErr() +} + +func (nw *NWriter) startWriting(src io.Reader) { + reader := NewSyncReader(src) + nw.wg.Add(nw.concurrency) + + for i := 0; i < nw.concurrency; i++ { + go func() { + defer nw.wg.Done() + + buffer := pool.Get().([]byte) + defer pool.Put(buffer) + + for nw.getErr() == nil { + n, err := reader.Read(buffer) + if err != nil && err != io.EOF { + nw.setErr(err) + return + } + if n > 0 { + partNum := atomic.AddInt64(&nw.partCount, 1) + if perr := nw.handler(int(partNum), int64(n), buffer[:n]); perr != nil { + nw.setErr(perr) + return + } + } + if err == io.EOF { + return + } + } + }() + } +} + +func (nw *NWriter) setErr(err error) { + nw.mu.Lock() + if nw.err == nil { + nw.err = err + } + nw.mu.Unlock() +} + +func (nw *NWriter) getErr() error { + nw.mu.Lock() + defer nw.mu.Unlock() + return nw.err +} diff --git a/internal/partedio/nwriter_test.go b/internal/partedio/nwriter_test.go new file mode 100644 index 0000000..f0a1ad6 --- /dev/null +++ b/internal/partedio/nwriter_test.go @@ -0,0 +1,238 @@ +package partedio + +import ( + "errors" + "io" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestNewNWriter(t *testing.T) { + tests := []struct { + name string + partSize int64 + handler PartHandler + concurrency int + wantErr bool + }{ + { + name: "valid parameters", + partSize: 100, + handler: func(int, int64, io.Reader) error { return nil }, + concurrency: 5, + wantErr: false, + }, + { + name: "zero part size", + partSize: 0, + handler: func(int, int64, io.Reader) error { return nil }, + concurrency: 5, + wantErr: true, + }, + { + name: "negative part size", + partSize: -1, + handler: func(int, int64, io.Reader) error { return nil }, + concurrency: 5, + wantErr: true, + }, + { + name: "nil handler", + partSize: 100, + handler: nil, + concurrency: 5, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := NewNWriter(tt.partSize, tt.concurrency, tt.handler) + if (err != nil) != tt.wantErr { + t.Errorf("NewNWriter() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestNWriterWrite(t *testing.T) { + tests := []struct { + name string + input string + partSize int64 + concurrency int + wantParts int64 + wantErr bool + handler func(t *testing.T) PartHandler + }{ + { + name: "write empty input", + input: "", + partSize: 4, + concurrency: 2, + wantParts: 0, + wantErr: false, + handler: func(t *testing.T) PartHandler { + return func(partNum int, size int64, r io.Reader) error { + t.Error("handler should not be called for empty input") + return nil + } + }, + }, + { + name: "write single part", + input: "test", + partSize: 4, + concurrency: 2, + wantParts: 1, + wantErr: false, + handler: func(t *testing.T) PartHandler { + return func(partNum int, size int64, r io.Reader) error { + data, err := io.ReadAll(r) + if err != nil { + t.Errorf("failed to read part: %v", err) + } + if string(data) != "test" { + t.Errorf("part data = %s, want %s", string(data), "test") + } + return nil + } + }, + }, + { + name: "write multiple parts", + input: "testdata", + partSize: 4, + concurrency: 2, + wantParts: 2, + wantErr: false, + handler: func(t *testing.T) PartHandler { + var mu sync.Mutex + parts := make(map[int]string) + return func(partNum int, size int64, r io.Reader) error { + data, err := io.ReadAll(r) + if err != nil { + t.Errorf("failed to read part: %v", err) + } + mu.Lock() + parts[partNum] = string(data) + mu.Unlock() + return nil + } + }, + }, + { + name: "handler returns error", + input: "test", + partSize: 4, + concurrency: 2, + wantParts: 1, + wantErr: true, + handler: func(t *testing.T) PartHandler { + return func(partNum int, size int64, r io.Reader) error { + return errors.New("handler error") + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var processedParts int64 + handler := func(partNum int, size int64, r io.Reader) error { + atomic.AddInt64(&processedParts, 1) + return tt.handler(t)(partNum, size, r) + } + + w, err := NewNWriter(tt.partSize, tt.concurrency, handler) + if err != nil { + t.Fatalf("failed to create writer: %v", err) + } + + _, err = w.Write([]byte(tt.input)) + if err != nil && !tt.wantErr { + t.Errorf("Write() unexpected error = %v", err) + } + + err = w.Close() + if (err != nil) != tt.wantErr { + t.Errorf("Close() error = %v, wantErr %v", err, tt.wantErr) + } + + if processedParts != tt.wantParts { + t.Errorf("processed %d parts, want %d", processedParts, tt.wantParts) + } + }) + } +} + +func TestNWriterConcurrentWrites(t *testing.T) { + input := strings.Repeat("test", 1000) + partSize := int64(4) + concurrency := 5 + + var maxConcurrent int32 + var currentConcurrent int32 + var mu sync.Mutex + processed := make(map[int]bool) + + handler := func(partNum int, size int64, r io.Reader) error { + current := atomic.AddInt32(¤tConcurrent, 1) + defer atomic.AddInt32(¤tConcurrent, -1) + + for { + cur := atomic.LoadInt32(&maxConcurrent) + if current > cur { + if atomic.CompareAndSwapInt32(&maxConcurrent, cur, current) { + break + } + continue + } + break + } + + time.Sleep(time.Millisecond) + + mu.Lock() + if processed[partNum] { + mu.Unlock() + t.Errorf("part %d processed multiple times", partNum) + return nil + } + processed[partNum] = true + mu.Unlock() + + return nil + } + + w, err := NewNWriter(partSize, concurrency, handler) + if err != nil { + t.Fatalf("failed to create writer: %v", err) + } + + _, err = w.Write([]byte(input)) + if err != nil { + t.Errorf("Write() error = %v", err) + } + + err = w.Close() + if err != nil { + t.Errorf("Close() error = %v", err) + } + + if maxConcurrent != int32(concurrency) { + t.Errorf("max concurrent executions = %d, want %d", maxConcurrent, concurrency) + } + + expectedParts := int64(len(input)) / partSize + if len(input)%int(partSize) > 0 { + expectedParts++ + } + + if int64(len(processed)) != expectedParts { + t.Errorf("processed %d parts, want %d", len(processed), expectedParts) + } +} diff --git a/internal/partedio/pool.go b/internal/partedio/pool.go new file mode 100644 index 0000000..a87af94 --- /dev/null +++ b/internal/partedio/pool.go @@ -0,0 +1 @@ +package partedio diff --git a/internal/partedio/reader.go b/internal/partedio/reader.go new file mode 100644 index 0000000..64964e1 --- /dev/null +++ b/internal/partedio/reader.go @@ -0,0 +1,141 @@ +package partedio + +import "io" + +type PartReader interface { + GetSize() int + GetReader(start, end int) (io.ReadCloser, error) +} + +type PartReaders []PartReader + +type Reader struct { + parts []PartReader + pos int64 + curIdx int + closed bool + reader io.ReadCloser + size int64 + + partStarts []int64 + partEnds []int64 +} + +func NewReader(parts PartReaders, pos int64) (*Reader, error) { + if len(parts) == 0 { + return nil, ErrNoParts + } + + partStarts := make([]int64, len(parts)) + partEnds := make([]int64, len(parts)) + + var offset int64 + for i, part := range parts { + partStarts[i] = offset + partEnds[i] = offset + int64(part.GetSize()) - 1 + offset = partEnds[i] + 1 + } + + if pos > offset { + return nil, io.EOF + } + + startIdx := 0 + for i := range parts { + if pos <= partEnds[i] { + startIdx = i + break + } + } + + return &Reader{ + parts: parts[startIdx:], + partStarts: partStarts[startIdx:], + partEnds: partEnds[startIdx:], + pos: pos, + size: offset, + curIdx: 0, + }, nil +} + +func (r *Reader) Read(p []byte) (int, error) { + if r.closed { + return 0, ErrClosed + } + + if len(p) == 0 { + return 0, nil + } + + if r.reader == nil { + if err := r.readNextPart(); err != nil { + return 0, err + } + } + + var totalRead int + for totalRead < len(p) { + nr, err := r.reader.Read(p[totalRead:]) + totalRead += nr + r.pos += int64(nr) + + if err == nil { + continue + } + + if err == io.EOF { + r.curIdx++ + if r.curIdx >= len(r.parts) { + return totalRead, io.EOF + } + + if err = r.readNextPart(); err != nil { + return totalRead, err + } + continue + } + + return totalRead, err + } + + return totalRead, nil +} + +func (r *Reader) Close() error { + if r.closed { + return ErrClosed + } + + var err error + if r.reader != nil { + err = r.reader.Close() + } + + r.closed = true + r.reader = nil + r.parts = nil // Help GC + r.partStarts = nil + r.partEnds = nil + return err +} + +func (r *Reader) readNextPart() error { + if r.reader != nil { + if err := r.reader.Close(); err != nil { + return err + } + } + + start := 0 + if r.pos > r.partStarts[r.curIdx] { + start = int(r.pos - r.partStarts[r.curIdx]) + } + + reader, err := r.parts[r.curIdx].GetReader(start, r.parts[r.curIdx].GetSize()-1) + if err != nil { + return err + } + + r.reader = reader + return nil +} diff --git a/internal/partedio/reader_test.go b/internal/partedio/reader_test.go new file mode 100644 index 0000000..39802b2 --- /dev/null +++ b/internal/partedio/reader_test.go @@ -0,0 +1,213 @@ +package partedio + +import ( + "bytes" + "errors" + "io" + "testing" +) + +type mockReader struct { + data []byte + offset int +} + +func (m *mockReader) Read(p []byte) (n int, err error) { + if m.offset >= len(m.data) { + return 0, io.EOF + } + n = copy(p, m.data[m.offset:]) + m.offset += n + return n, nil +} + +func (m *mockReader) Close() error { + return nil +} + +// mockPart implements PartReader interface +type mockPart struct { + size int + reader func(start, end int) (io.ReadCloser, error) +} + +func (m *mockPart) GetSize() int { + return m.size +} + +func (m *mockPart) GetReader(start, end int) (io.ReadCloser, error) { + return m.reader(start, end) +} + +func TestNewReader(t *testing.T) { + tests := []struct { + name string + parts []PartReader + pos int64 + wantErr error + }{ + { + name: "empty parts", + parts: []PartReader{}, + pos: 0, + wantErr: ErrNoParts, + }, + { + name: "position beyond end", + parts: []PartReader{ + &mockPart{size: 100}, + &mockPart{size: 50}, + }, + pos: 200, + wantErr: io.EOF, + }, + { + name: "valid parts and position", + parts: []PartReader{ + &mockPart{size: 100}, + &mockPart{size: 50}, + }, + pos: 0, + wantErr: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := NewReader(tt.parts, tt.pos) + if !errors.Is(err, tt.wantErr) { + t.Errorf("NewReader() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestReadAcrossParts(t *testing.T) { + part1Data := bytes.Repeat([]byte("a"), 100) + part2Data := bytes.Repeat([]byte("b"), 50) + part3Data := bytes.Repeat([]byte("c"), 75) + + parts := []PartReader{ + &mockPart{ + size: 100, + reader: func(start, end int) (io.ReadCloser, error) { + return &mockReader{data: part1Data[start : end+1], offset: 0}, nil + }, + }, + &mockPart{ + size: 50, + reader: func(start, end int) (io.ReadCloser, error) { + return &mockReader{data: part2Data[start : end+1], offset: 0}, nil + }, + }, + &mockPart{ + size: 75, + reader: func(start, end int) (io.ReadCloser, error) { + return &mockReader{data: part3Data[start : end+1], offset: 0}, nil + }, + }, + } + + tests := []struct { + name string + pos int64 + readSize int + want string + wantErr error + }{ + { + name: "read within first part", + pos: 0, + readSize: 50, + want: string(bytes.Repeat([]byte("a"), 50)), + wantErr: nil, + }, + { + name: "read across first and second parts", + pos: 90, + readSize: 30, + want: string(append(bytes.Repeat([]byte("a"), 10), bytes.Repeat([]byte("b"), 20)...)), + wantErr: nil, + }, + { + name: "read to EOF", + pos: 200, + readSize: 30, + want: string(bytes.Repeat([]byte("c"), 25)), + wantErr: io.EOF, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r, err := NewReader(parts, tt.pos) + if err != nil { + t.Fatalf("NewReader() error = %v", err) + } + defer r.Close() + + buf := make([]byte, tt.readSize) + n, err := r.Read(buf) + if !errors.Is(err, tt.wantErr) { + t.Errorf("Read() error = %v, wantErr %v", err, tt.wantErr) + } + + got := string(buf[:n]) + if got != tt.want { + t.Errorf("Read() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestReaderClose(t *testing.T) { + part := &mockPart{ + size: 100, + reader: func(start, end int) (io.ReadCloser, error) { + return &mockReader{data: make([]byte, end-start+1), offset: 0}, nil + }, + } + + r, err := NewReader([]PartReader{part}, 0) + if err != nil { + t.Fatalf("NewReader() error = %v", err) + } + + buf := make([]byte, 10) + _, err = r.Read(buf) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + + if err := r.Close(); err != nil { + t.Errorf("First Close() error = %v", err) + } + + if err := r.Close(); !errors.Is(err, ErrClosed) { + t.Errorf("Second Close() error = %v, want %v", err, ErrClosed) + } + + if _, err := r.Read(buf); !errors.Is(err, ErrClosed) { + t.Errorf("Read() after Close error = %v, want %v", err, ErrClosed) + } +} + +func TestEmptyRead(t *testing.T) { + part := &mockPart{ + size: 100, + reader: func(start, end int) (io.ReadCloser, error) { + return &mockReader{data: make([]byte, end-start+1), offset: 0}, nil + }, + } + + r, err := NewReader([]PartReader{part}, 0) + if err != nil { + t.Fatalf("NewReader() error = %v", err) + } + defer r.Close() + + n, err := r.Read([]byte{}) + if n != 0 || err != nil { + t.Errorf("Read() empty buffer got = %v, %v, want 0, nil", n, err) + } +} diff --git a/internal/partedio/sreader.go b/internal/partedio/sreader.go new file mode 100644 index 0000000..8352e86 --- /dev/null +++ b/internal/partedio/sreader.go @@ -0,0 +1,32 @@ +package partedio + +import ( + "io" + "sync" +) + +type SyncReader struct { + reader io.Reader + mu sync.Mutex +} + +func NewSyncReader(r io.Reader) io.Reader { + return &SyncReader{r, sync.Mutex{}} +} + +func (br *SyncReader) Read(p []byte) (int, error) { + br.mu.Lock() + defer br.mu.Unlock() + + currReadIdx := 0 + // Loop until p is full + for currReadIdx < len(p) { + n, err := br.reader.Read(p[currReadIdx:]) + currReadIdx += n + if err != nil { + return currReadIdx, err + } + } + + return currReadIdx, nil +} diff --git a/internal/partedio/writer.go b/internal/partedio/writer.go new file mode 100644 index 0000000..511c157 --- /dev/null +++ b/internal/partedio/writer.go @@ -0,0 +1,115 @@ +package partedio + +import "io" + +type Processor func(size int64, reader io.Reader) error + +type Writer struct { + partSize int64 + totalSize int64 + totalWritten int64 // total bytes written + processed int64 // bytes in current chunk + closed bool + errCh chan error + pwriter *io.PipeWriter + processor Processor + err error +} + +func NewWriter(totalSize int64, partSize int64, processor Processor) *Writer { + return &Writer{ + totalSize: totalSize, + partSize: partSize, + processor: processor, + errCh: make(chan error), + } +} + +func (w *Writer) Write(p []byte) (int, error) { + if w.closed { + return 0, ErrClosed + } + + if w.err != nil { + return 0, w.err + } + + if w.pwriter == nil { + w.startNextPart() + } + + total := len(p) + for len(p) > 0 { + if w.processed+int64(len(p)) > w.partSize { + n, err := w.pwriter.Write(p[:w.partSize-w.processed]) + if err != nil { + return total - len(p), err + } + w.totalWritten += int64(n) + if err = w.finishPart(true); err != nil { + return total - len(p), err + } + p = p[n:] + } else { + n, err := w.pwriter.Write(p) + if err != nil { + return total - len(p), err + } + w.processed += int64(n) + w.totalWritten += int64(n) + p = p[n:] + } + } + return total, nil +} + +func (w *Writer) Close() error { + if w.closed { + return ErrClosed + } + w.closed = true + if w.pwriter != nil { + return w.finishPart(false) + } + return nil +} + +func (w *Writer) finishPart(startNew bool) error { + if err := w.pwriter.Close(); err != nil { + return err + } + if err := <-w.errCh; err != nil { + return err + } + if startNew { + w.startNextPart() + } + return nil +} + +func (w *Writer) startNextPart() { + if !w.closed { + reader, writer := io.Pipe() + w.pwriter = writer + w.processed = 0 + + remainingSize := w.totalSize - w.totalWritten + partSize := w.partSize + if remainingSize < w.partSize { + partSize = remainingSize + } + + go func() { + err := w.processor(partSize, reader) + if err != nil { + w.err = err + _ = reader.CloseWithError(err) + } + // Sometime processor is dumbfuck + // neither read all the data nor return error + _, _ = io.Copy(io.Discard, reader) + _ = reader.Close() + w.errCh <- w.err + }() + } +} diff --git a/internal/partedio/writer_test.go b/internal/partedio/writer_test.go new file mode 100644 index 0000000..e146185 --- /dev/null +++ b/internal/partedio/writer_test.go @@ -0,0 +1,261 @@ +package partedio + +import ( + "bytes" + "errors" + "io" + "testing" +) + +func TestWriterSinglePart(t *testing.T) { + var collected []byte + var receivedSize int64 + + processor := func(size int64, reader io.Reader) error { + receivedSize = size + data, err := io.ReadAll(reader) + if err != nil { + return err + } + collected = append(collected, data...) + return nil + } + + w := NewWriter(10, 10, processor) + + input := []byte("helloworld") + n, err := w.Write(input) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + if n != len(input) { + t.Errorf("Expected to write %d bytes, wrote %d", len(input), n) + } + + err = w.Close() + if err != nil { + t.Fatalf("Close failed: %v", err) + } + + if !bytes.Equal(collected, input) { + t.Errorf("Expected %q, got %q", input, collected) + } + if receivedSize != 10 { + t.Errorf("Expected size 10, got %d", receivedSize) + } +} + +func TestWriterMultiPart(t *testing.T) { + var parts [][]byte + var sizes []int64 + + processor := func(size int64, reader io.Reader) error { + data, err := io.ReadAll(reader) + if err != nil { + return err + } + parts = append(parts, data) + sizes = append(sizes, size) + return nil + } + + w := NewWriter(10, 4, processor) + + input := []byte("helloworld") + n, err := w.Write(input) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + if n != len(input) { + t.Errorf("Expected to write %d bytes, wrote %d", len(input), n) + } + + err = w.Close() + if err != nil { + t.Fatalf("Close failed: %v", err) + } + + expectedParts := 3 + if len(parts) != expectedParts { + t.Fatalf("Expected %d parts, got %d", expectedParts, len(parts)) + } + + expectedParts1 := []byte("hell") + expectedParts2 := []byte("owor") + expectedParts3 := []byte("ld") + + if !bytes.Equal(parts[0], expectedParts1) { + t.Errorf("Part 1: expected %q, got %q", expectedParts1, parts[0]) + } + if !bytes.Equal(parts[1], expectedParts2) { + t.Errorf("Part 2: expected %q, got %q", expectedParts2, parts[1]) + } + if !bytes.Equal(parts[2], expectedParts3) { + t.Errorf("Part 3: expected %q, got %q", expectedParts3, parts[2]) + } + + if sizes[0] != 4 || sizes[1] != 4 || sizes[2] != 2 { + t.Errorf("Unexpected sizes: %v", sizes) + } +} + +func TestWriterProcessorError(t *testing.T) { + expectedError := errors.New("processor error") + processor := func(size int64, reader io.Reader) error { + return expectedError + } + + w := NewWriter(10, 5, processor) + + _, err := w.Write([]byte("hello")) + if !errors.Is(err, expectedError) { + t.Errorf("Expected error %v, got %v", expectedError, err) + } +} + +func TestWriterWriteAfterClose(t *testing.T) { + processor := func(size int64, reader io.Reader) error { + _, err := io.ReadAll(reader) + return err + } + + w := NewWriter(10, 5, processor) + + err := w.Close() + if err != nil { + t.Fatalf("Close failed: %v", err) + } + + _, err = w.Write([]byte("hello")) + if !errors.Is(err, ErrClosed) { + t.Errorf("Expected ErrClosed, got %v", err) + } +} + +func TestWriterLargeWrite(t *testing.T) { + var totalReceived int + + processor := func(size int64, reader io.Reader) error { + data, err := io.ReadAll(reader) + if err != nil { + return err + } + totalReceived += len(data) + return nil + } + + totalSize := int64(1000) + partSize := int64(100) + w := NewWriter(totalSize, partSize, processor) + + data := make([]byte, 500) + for i := range data { + data[i] = byte(i % 256) + } + + for i := 0; i < 2; i++ { + n, err := w.Write(data) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + if n != len(data) { + t.Errorf("Expected to write %d bytes, wrote %d", len(data), n) + } + } + + err := w.Close() + if err != nil { + t.Fatalf("Close failed: %v", err) + } + + if totalReceived != 1000 { + t.Errorf("Expected to receive 1000 bytes, got %d", totalReceived) + } +} + +func TestWriterEarlyClose(t *testing.T) { + var parts [][]byte + var sizes []int64 + + processor := func(size int64, reader io.Reader) error { + data, err := io.ReadAll(reader) + if err != nil { + return err + } + parts = append(parts, data) + sizes = append(sizes, size) + return nil + } + + w := NewWriter(10, 4, processor) + + input := []byte("hello") + n, err := w.Write(input) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + if n != len(input) { + t.Errorf("Expected to write %d bytes, wrote %d", len(input), n) + } + + err = w.Close() + if err != nil { + t.Fatalf("Close failed: %v", err) + } + + expectedParts := 2 + if len(parts) != expectedParts { + t.Fatalf("Expected %d parts, got %d", expectedParts, len(parts)) + } + + expectedPart1 := []byte("hell") + expectedPart2 := []byte("o") + if !bytes.Equal(parts[0], expectedPart1) { + t.Errorf("Part 1: expected %q, got %q", expectedPart1, parts[0]) + } + if !bytes.Equal(parts[1], expectedPart2) { + t.Errorf("Part 2: expected %q, got %q", expectedPart2, parts[1]) + } + + if sizes[0] != 4 || sizes[1] != 4 { + t.Errorf("Unexpected sizes: %v", sizes) + } + + _, err = w.Write([]byte("world")) + if !errors.Is(err, ErrClosed) { + t.Errorf("Expected ErrClosed, got %v", err) + } +} + +func TestWriterSizeMismatch(t *testing.T) { + w := NewWriter(5, 10, func(size int64, r io.Reader) error { + _, _ = io.ReadAll(r) + return nil + }) + + data := []byte("hello") + _, err := w.Write(data) + if err != nil { + t.Fatal(err) + } + + if err := w.Close(); err != nil { + t.Fatal(err) + } +} + +func TestWriterProcessorNotReading(t *testing.T) { + w := NewWriter(5, 10, func(size int64, r io.Reader) error { + return nil + }) + + data := []byte("hello") + _, err := w.Write(data) + if err != nil { + t.Fatal(err) + } + + if err := w.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/types.go b/internal/types.go new file mode 100644 index 0000000..a2bdb50 --- /dev/null +++ b/internal/types.go @@ -0,0 +1,31 @@ +package internal + +import ( + "io" + "time" +) + +type MetaFileSystem interface { + Name() string + Create(path string, isDir bool) (*Node, error) + Stat(path string) (*Node, error) + Ls(path string, limit int, offset int) ([]Node, error) + Chtimes(path string, mtime time.Time) error + Touch(path string) error + Mkdir(path string) error + MkdirAll(path string) error + Remove(path string) error + RemoveAll(path string) error + Rename(oldpath, newpath string) error + Close() error + + // Sync - update node's + Sync(path string, size int64) error +} + +type StorageDriver interface { + GetReader(fileId string, pos int64) (io.ReadCloser, error) + GetWriter(fileId string) (io.WriteCloser, error) + GetSize(fileId string) (int64, error) + Truncate(fileId string) error +} diff --git a/internal/version.go b/internal/version.go new file mode 100644 index 0000000..a0ea602 --- /dev/null +++ b/internal/version.go @@ -0,0 +1,35 @@ +package internal + +import ( + "fmt" + "runtime/debug" +) + +func Version() string { + var revision string + var modified bool + + bi, ok := debug.ReadBuildInfo() + if ok { + for _, s := range bi.Settings { + switch s.Key { + case "vcs.revision": + revision = s.Value + case "vcs.modified": + if s.Value == "true" { + modified = true + } + } + } + } + + if revision == "" { + return "unavailable" + } + + if modified { + return fmt.Sprintf("%s-dirty", revision) + } + + return revision +} diff --git a/pkg/logfs.go b/pkg/logfs.go new file mode 100644 index 0000000..bd705d2 --- /dev/null +++ b/pkg/logfs.go @@ -0,0 +1,250 @@ +package pkg + +import ( + "io" + "os" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/afero" +) + +type LogFile struct { + src afero.File + lengthRead int64 + lengthWritten int64 + name string + logger *zerolog.Logger +} + +type LogFS struct { + src afero.Fs + logger *zerolog.Logger +} + +func NewLogFS(src afero.Fs) afero.Fs { + logger := log.With().Str("component", "filesystem").Logger() + return &LogFS{src: src, logger: &logger} +} + +func (lf *LogFS) logOperation(err error, operation string, fields map[string]interface{}) { + event := lf.logger.Debug() + if err != nil { + event = lf.logger.Error().Err(err) + } + + for key, value := range fields { + event = event.Interface(key, value) + } + + event.Str("operation", operation).Send() +} + +func (lff *LogFile) logOperation(err error, operation string, fields map[string]interface{}) { + event := lff.logger.Debug() + if err != nil { + event = lff.logger.Error().Err(err) + } + + fields["name"] = lff.name + for key, value := range fields { + event = event.Interface(key, value) + } + + event.Str("operation", operation).Send() +} + +func (lf *LogFS) newLogFile(file afero.File, err error) (afero.File, error) { + if err != nil { + return file, err + } + return &LogFile{ + src: file, + name: file.Name(), + logger: lf.logger, + }, nil +} + +func (lf *LogFS) Create(name string) (afero.File, error) { + file, err := lf.src.Create(name) + lf.logOperation(err, "CREATE", map[string]interface{}{"name": name}) + return lf.newLogFile(file, err) +} + +func (lf *LogFS) Mkdir(name string, perm os.FileMode) error { + err := lf.src.Mkdir(name, perm) + lf.logOperation(err, "MKDIR", map[string]interface{}{ + "name": name, + "perm": perm, + }) + return err +} + +func (lf *LogFS) MkdirAll(path string, perm os.FileMode) error { + err := lf.src.MkdirAll(path, perm) + lf.logOperation(err, "MKDIR_ALL", map[string]interface{}{ + "path": path, + "perm": perm, + }) + return err +} + +func (lf *LogFS) Open(name string) (afero.File, error) { + file, err := lf.src.Open(name) + lf.logOperation(err, "OPEN", map[string]interface{}{"name": name}) + return lf.newLogFile(file, err) +} + +func (lf *LogFS) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + file, err := lf.src.OpenFile(name, flag, perm) + lf.logOperation(err, "OPEN_FILE", map[string]interface{}{ + "name": name, + "flag": flag, + "perm": perm, + }) + return lf.newLogFile(file, err) +} + +func (lf *LogFS) Remove(name string) error { + err := lf.src.Remove(name) + lf.logOperation(err, "REMOVE", map[string]interface{}{"name": name}) + return err +} + +func (lf *LogFS) RemoveAll(path string) error { + err := lf.src.RemoveAll(path) + lf.logOperation(err, "REMOVE_ALL", map[string]interface{}{"path": path}) + return err +} + +func (lf *LogFS) Rename(oldname, newname string) error { + err := lf.src.Rename(oldname, newname) + lf.logOperation(err, "RENAME", map[string]interface{}{ + "oldname": oldname, + "newname": newname, + }) + return err +} + +func (lff *LogFile) Close() error { + err := lff.src.Close() + lff.logOperation(err, "CLOSE", map[string]interface{}{ + "bytes_read": lff.lengthRead, + "bytes_written": lff.lengthWritten, + }) + return err +} + +func (lff *LogFile) Read(p []byte) (int, error) { + n, err := lff.src.Read(p) + if err == nil { + lff.lengthRead += int64(n) + } else if err != io.EOF { + lff.logOperation(err, "READ", nil) + } + return n, err +} + +func (lff *LogFile) ReadAt(p []byte, off int64) (int, error) { + n, err := lff.src.ReadAt(p, off) + if err == nil { + lff.lengthRead += int64(n) + } else if err != io.EOF { + lff.logOperation(err, "READ_AT", map[string]interface{}{"offset": off}) + } + return n, err +} + +func (lff *LogFile) Seek(offset int64, whence int) (int64, error) { + n, err := lff.src.Seek(offset, whence) + if err != nil { + lff.logOperation(err, "SEEK", map[string]interface{}{ + "offset": offset, + "whence": whence, + }) + } + return n, err +} + +func (lff *LogFile) Write(p []byte) (int, error) { + n, err := lff.src.Write(p) + if err == nil { + lff.lengthWritten += int64(n) + } else { + lff.logOperation(err, "WRITE", nil) + } + return n, err +} + +func (lff *LogFile) WriteAt(p []byte, off int64) (int, error) { + n, err := lff.src.WriteAt(p, off) + if err == nil { + lff.lengthWritten += int64(n) + } else { + lff.logOperation(err, "WRITE_AT", map[string]interface{}{"offset": off}) + } + return n, err +} + +func (lff *LogFile) WriteString(s string) (int, error) { + n, err := lff.src.WriteString(s) + if err == nil { + lff.lengthWritten += int64(n) + } else { + lff.logOperation(err, "WRITE_STRING", nil) + } + return n, err +} + +func (lff *LogFile) Readdir(count int) ([]os.FileInfo, error) { + info, err := lff.src.Readdir(count) + lff.logOperation(err, "READ_DIR", map[string]interface{}{"count": count}) + return info, err +} + +func (lff *LogFile) Readdirnames(n int) ([]string, error) { + names, err := lff.src.Readdirnames(n) + lff.logOperation(err, "READ_DIR_NAMES", map[string]interface{}{"count": n}) + return names, err +} + +func (lff *LogFile) Sync() error { + err := lff.src.Sync() + lff.logOperation(err, "SYNC", nil) + return err +} + +func (lff *LogFile) Truncate(size int64) error { + err := lff.src.Truncate(size) + lff.logOperation(err, "TRUNCATE", map[string]interface{}{"size": size}) + return err +} + +func (lf *LogFS) Name() string { + return lf.src.Name() +} + +func (lf *LogFS) Chmod(name string, mode os.FileMode) error { + return lf.src.Chmod(name, mode) +} + +func (lf *LogFS) Chtimes(name string, atime time.Time, mtime time.Time) error { + return lf.src.Chtimes(name, atime, mtime) +} + +func (lf *LogFS) Chown(name string, uid, gid int) error { + return lf.src.Chown(name, uid, gid) +} + +func (lf *LogFS) Stat(name string) (os.FileInfo, error) { + return lf.src.Stat(name) +} + +func (lff *LogFile) Name() string { + return lff.name +} + +func (lff *LogFile) Stat() (os.FileInfo, error) { + return lff.src.Stat() +} diff --git a/staticcheck.conf b/staticcheck.conf new file mode 100644 index 0000000..a444106 --- /dev/null +++ b/staticcheck.conf @@ -0,0 +1,2 @@ +checks = ["all", "-ST1000", "-U1000"] +initialisms = ["ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS", "SIP", "RTP", "AMQP", "TS"]