use decompressed data after getting unexpected EOF (#2)

* use decompressed data after getting unexpected EOF

* config refactor

---------

Co-authored-by: Nahum Perman <[email protected]>

Author: nahum-perman

build/package/Dockerfile

@@ -36,8 +36,9 @@ RUN setcap CAP_NET_BIND_SERVICE=+eip /go/bin/app
 # Use an unprivileged user.
 USER appuser
 
+ENV APPSEC_MODE "standalone"
 # Port on which the service will be exposed.
-EXPOSE 8080
+EXPOSE 80
 
 # Run the app binary.
 ENTRYPOINT ["/go/bin/app"]

cmd/server/main.go

@@ -18,9 +18,9 @@ import (
 	"os"
 	"time"
 
+	"openappsec.io/log"
 	"openappsec.io/smartsync-service/internal/app"
 	"openappsec.io/smartsync-service/internal/app/ingector"
-	"openappsec.io/log"
 )
 
 const (
@@ -35,7 +35,7 @@ const (
 func main() {
 	initCtx, initCancel := context.WithTimeout(context.Background(), initTimeout*time.Second)
 
-	isStandAloneMode := os.Getenv(envKeyMode) == "stand-alone"
+	isStandAloneMode := os.Getenv(envKeyMode) == "standalone"
 	var app *app.App
 	var err error
 	if isStandAloneMode {

configs/bootstrapConfig.yaml

@@ -1,5 +1,5 @@
 server:
-  port: 8080
+  port: 80
   timeout: "15s"
   configurationServer: "" # configuration server can be either etcd or empty
 log:

go.mod

@@ -2,6 +2,18 @@ module openappsec.io/smartsync-service
 
 go 1.18
 
+replace (
+	openappsec.io/configuration => ./dependencies/openappsec.io/configuration
+	openappsec.io/ctxutils => ./dependencies/openappsec.io/ctxutils
+	openappsec.io/errors => ./dependencies/openappsec.io/errors
+	openappsec.io/health => ./dependencies/openappsec.io/health
+	openappsec.io/httputils => ./dependencies/openappsec.io/httputils
+	openappsec.io/kafka => ./dependencies/openappsec.io/kafka
+	openappsec.io/log => ./dependencies/openappsec.io/log
+	openappsec.io/redis => ./dependencies/openappsec.io/redis
+	openappsec.io/tracer => ./dependencies/openappsec.io/tracer
+)
+
 require (
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/google/uuid v1.1.2
@@ -49,15 +61,3 @@ require (
 	gopkg.in/ini.v1 v1.64.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
-
-replace (
-	openappsec.io/configuration => ./dependencies/openappsec.io/configuration
-	openappsec.io/ctxutils => ./dependencies/openappsec.io/ctxutils
-	openappsec.io/errors => ./dependencies/openappsec.io/errors
-	openappsec.io/health => ./dependencies/openappsec.io/health
-	openappsec.io/httputils => ./dependencies/openappsec.io/httputils
-	openappsec.io/kafka => ./dependencies/openappsec.io/kafka
-	openappsec.io/log => ./dependencies/openappsec.io/log
-	openappsec.io/redis => ./dependencies/openappsec.io/redis
-	openappsec.io/tracer => ./dependencies/openappsec.io/tracer
-)

internal/pkg/db/s3/s3.go

@@ -19,15 +19,15 @@ import (
 	"context"
 	"encoding/json"
 	"encoding/xml"
+	errs "errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
-	"time"
-
 	"openappsec.io/smartsync-service/models"
+	"time"
 
 	"openappsec.io/ctxutils"
 	"openappsec.io/errors"
@@ -39,8 +39,10 @@ const (
 	traceClientTimeout = 2 * time.Minute
 
 	//conf Key Reverse Proxy
-	rp        = "rp"
-	rpBaseURL = rp + ".baseUrl"
+	rp                   = "rp"
+	rpBaseURL            = rp + ".baseUrl"
+	sharedStorageKey     = "shared_storage"
+	sharedStorageHostKey = sharedStorageKey + ".host"
 
 	tenantIDHeader         = "x-tenant-id"
 	headerKeyTraceID       = "X-Trace-Id"
@@ -71,7 +73,13 @@ func NewAdapter(c Configuration) (*Adapter, error) {
 
 // initialize the adapter
 func (a *Adapter) initialize(c Configuration) error {
-	baseURL, err := c.GetString(rpBaseURL)
+	var baseURL string
+	host, err := c.GetString(sharedStorageHostKey)
+	if err != nil {
+		baseURL, err = c.GetString(rpBaseURL)
+	} else {
+		baseURL = fmt.Sprintf("http://%s/api", host)
+	}
 	if err != nil {
 		return errors.Wrapf(err, "failed to get reverse proxy baseURL from %v", rpBaseURL)
 	}
@@ -84,7 +92,7 @@ func (a *Adapter) initialize(c Configuration) error {
 	return nil
 }
 
-//GetFileRaw - download file, return raw data and true if data was decompressed
+// GetFileRaw - download file, return raw data and true if data was decompressed
 func (a *Adapter) GetFileRaw(ctx context.Context, tenantID string, path string) ([]byte, bool, error) {
 	if path[:1] != "/" {
 		path = "/" + path
@@ -108,7 +116,7 @@ func (a *Adapter) GetFileRaw(ctx context.Context, tenantID string, path string)
 		}
 		decompressed, err := ioutil.ReadAll(compressor)
 		defer compressor.Close()
-		if err != nil {
+		if err != nil && !errs.Is(err, io.ErrUnexpectedEOF) {
 			log.WithContext(ctx).Warnf("failed to decompress, err: %v, is EOF: %v", err, err == io.EOF)
 			return []byte{}, false, errors.Wrapf(err, "failed to decompress %v", string(resp))
 		}
@@ -121,7 +129,7 @@ func (a *Adapter) GetFileRaw(ctx context.Context, tenantID string, path string)
 	return resp, false, nil
 }
 
-//GetFile - download file and unmarshal to out return true is data was decompressed
+// GetFile - download file and unmarshal to out return true is data was decompressed
 func (a *Adapter) GetFile(ctx context.Context, tenantID string, path string, out interface{}) (bool, error) {
 	data, isCompressed, err := a.GetFileRaw(ctx, tenantID, path)
 	if err != nil {