Usage Guide Go (golang)¶

// Copyright (C) 2017 CoSoSys S.R.L.
//       sensitivity.io
//       +40-264-593 110
//       Ovidiu CICAL - ovidiu.cical@sensitivity.io
// api/scanner.go

package api

/*
#cgo CFLAGS: -DSENSITIVITYIO_SHARED -I. -I/opt/cososys/include
#cgo LDFLAGS: -Wl,-rpath,/opt/cososys/lib64 -L/opt/cososys/lib64 -lsensitivityio_base -lsensitivityio_c_base -lsensitivityio_c_license -lsensitivityio_c_sds

#include <sensitivityio/c_sds/scanner.h>
#include <sensitivityio/c_sds/scanner_settings_loader.h>
*/
import "C"

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/mux"

	log "github.com/alecthomas/log4go"

	"sensitivity.io/api/model"
	"sensitivity.io/api/utils"
)

func InitScanner() {
	BaseRoutes.Scanner.Handle("/data", ApiAccountRequired(scanData)).Methods("POST")              // scan from upload data
	BaseRoutes.Scanner.Handle("/upload_file", ApiAccountRequired(scanUploadFile)).Methods("POST") // scan from upload file

	BaseRoutes.NeedScanner.Handle("/", ApiAccountRequired(getScanner)).Methods("GET") // get scanner by Id
}

var dumpJsonToFile = true

func scanData(c *Context, w http.ResponseWriter, r *http.Request) {

	// Scan start time in milliseconds
	scanStartTimestamp := utils.MillisFromTime(time.Now())

	params := mux.Vars(r)
	// return if provided data for scanner is not valid
	if _, valid := validScannerData(c, params); !valid {
		c.SetInvalidParam("scanData", "params")
		return
	}
	projectId := params["project_id"]
	appId := params["app_id"]

	if r.ContentLength > *utils.Cfg.FileSettings.MaxFileSize {
		c.Err = model.NewAppError("scannerCaller", "api.scanner.scan_data.too_large.error", nil, "")
		c.Err.StatusCode = http.StatusRequestEntityTooLarge
		return
	}

	if err := r.ParseMultipartForm(*utils.Cfg.FileSettings.MaxFileSize); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	m := r.MultipartForm

	fields := m.Value
	if len(fields["data"]) == 0 {
		c.SetInvalidParam("scanData", "data fields")
		return
	}

	c.LogAdminAction(
		"Scanner",  // module
		"",         // objectId
		"",         // identifier
		"ScanData", // action
		"",         // beforeDesc
		"",         // afterDesc
		fmt.Sprintf("ProjectId=%s, AppId=%s", projectId, appId)) // extraInfo

	var data string
	for _, part := range fields["data"] {
		data += fmt.Sprintf("%v ", part)
	}

	var verbose bool
	if len(r.FormValue("verbose")) > 0 {
		verbose, _ = strconv.ParseBool(strings.TrimSpace(r.FormValue("verbose")))
	} else {
		verbose = true
	}
	stopAtFirst, _ := strconv.ParseBool(strings.TrimSpace(r.FormValue("stop_at_first")))
	stopAt, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("stop_at")))
	var enableThreatHandler bool
	if stopAt > 0 {
		enableThreatHandler = true
	}
	var maskThreats bool
	if len(r.FormValue("mask_threats")) > 0 {
		maskThreats, _ = strconv.ParseBool(strings.TrimSpace(r.FormValue("mask_threats")))
	}

	scannerSettings := &model.ScannerSettings{
		ThreatHandler: enableThreatHandler,
		Verbose:       verbose,
		StopAtFirst:   stopAtFirst,
		StopAt:        stopAt,
		MaskThreats:   maskThreats,
		License:       new(string),
		Settings:      new(string),
	}

	// get App Settings
	if license, settings, valid := getAppDataForScanner(c, projectId, appId); !valid {
		return
	} else {
		*scannerSettings.License = license
		*scannerSettings.Settings = settings.ToJson()
	}
	scannerSettings.SetDefaults()

	if pScanner, pSettingsLoader, err := utils.InitSensitivityIoScanner(scannerSettings); err != nil {
		utils.UnregisterAndDestroy(pScanner, pSettingsLoader)
		c.Err = err
		c.Err.StatusCode = http.StatusServiceUnavailable
		return
	} else {
		scannerId := model.NewRandomHex(model.SCANNER_ID_LENGTH)
		resStruct := &model.ThreatResponse{ScannerId: scannerId}

		quitChan := make(chan int)
		defer close(quitChan)

		cchan := utils.ScanData(data, quitChan, scannerSettings, pScanner)
		defer utils.UnregisterAndDestroy(pScanner, pSettingsLoader)

		if cresult := <-cchan; cresult.Err != nil {
			fmt.Println(cresult.Err)
		} else {
			resStruct.ThreatResults = append(resStruct.ThreatResults, cresult)
			res := []byte(resStruct.ToJson())
			if dumpJsonToFile {
				scanLogger := utils.InitScanLogger(c.AccountId, projectId, appId, scannerId)
				logFile, lfErr := scanLogger.OpenLogFile()
				if lfErr != nil {
					return
				}
				defer logFile.Close()
				scanLogger.WriteToFile(logFile, resStruct, scanStartTimestamp)
			}
			w.Write(res)
		}
	}
}

func scanUploadFile(c *Context, w http.ResponseWriter, r *http.Request) {

	// Scan start time in milliseconds
	scanStartTimestamp := utils.MillisFromTime(time.Now())

	params := mux.Vars(r)
	// return if provided data for scanner is not valid
	if _, valid := validScannerData(c, params); !valid {
		c.SetInvalidParam("scanUploadFile", "params")
		return
	}
	projectId := params["project_id"]
	appId := params["app_id"]

	if r.ContentLength > *utils.Cfg.FileSettings.MaxFileSize {
		c.Err = model.NewAppError("scanUploadFile", "api.scanner.scan_file.too_large", nil, "")
		c.Err.StatusCode = http.StatusRequestEntityTooLarge
		return
	}

	if err := r.ParseMultipartForm(*utils.Cfg.FileSettings.MaxFileSize); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	m := r.MultipartForm

	files := m.File
	if len(files) == 0 {
		c.SetInvalidParam("scanUploadFile", "file")
		return
	}

	c.LogAdminAction(
		"Scanner",        // module
		"",               // objectId
		"",               // identifier
		"ScanUploadFile", // action
		"",               // beforeDesc
		"",               // afterDesc
		fmt.Sprintf("ProjectId=%s, AppId=%s", projectId, appId)) // extraInfo

	var verbose bool
	if len(r.FormValue("verbose")) > 0 {
		verbose, _ = strconv.ParseBool(strings.TrimSpace(r.FormValue("verbose")))
	} else {
		verbose = true
	}
	stopAtFirst, _ := strconv.ParseBool(strings.TrimSpace(r.FormValue("stop_at_first")))
	stopAt, _ := strconv.Atoi(strings.TrimSpace(r.FormValue("stop_at")))
	var enableThreatHandler bool
	if stopAt > 0 {
		enableThreatHandler = true
	}
	var maskThreats bool
	if len(r.FormValue("mask_threats")) > 0 {
		maskThreats, _ = strconv.ParseBool(strings.TrimSpace(r.FormValue("mask_threats")))
	}

	scannerSettings := &model.ScannerSettings{
		ThreatHandler: enableThreatHandler,
		Verbose:       verbose,
		StopAtFirst:   stopAtFirst,
		StopAt:        stopAt,
		MaskThreats:   maskThreats,
		License:       new(string),
		Settings:      new(string),
	}

	// get App Settings
	if license, settings, valid := getAppDataForScanner(c, projectId, appId); !valid {
		return
	} else {
		*scannerSettings.License = license
		*scannerSettings.Settings = settings.ToJson()
	}
	scannerSettings.SetDefaults()

	if pScanner, pSettingsLoader, err := utils.InitSensitivityIoScanner(scannerSettings); err != nil {
		utils.UnregisterAndDestroy(pScanner, pSettingsLoader)
		c.Err = err
		c.Err.StatusCode = http.StatusServiceUnavailable
		return
	} else {
		scannerId := model.NewRandomHex(model.SCANNER_ID_LENGTH)
		resStruct := &model.ThreatResponse{ScannerId: scannerId}

		defer utils.UnregisterAndDestroy(pScanner, pSettingsLoader)

		for _, fileHeader := range files["files"] {
			file, fileErr := fileHeader.Open()
			if fileErr != nil {
				resStruct.Details = append(resStruct.Details, fileErr.Error())
				continue
			}
			defer file.Close()

			buf := bytes.NewBuffer(nil)
			io.Copy(buf, file)
			data := buf.Bytes()

			// START - Scan file from Disk with saving it locally
			info, err := utils.DoUploadFile(c.AccountId, projectId, appId, fileHeader.Filename, data)
			if err != nil {
				resStruct.Details = append(resStruct.Details, err.Error())
				continue
			}
			info.Name = filepath.Base(fileHeader.Filename)

			if !model.IsFileExAllowed(filepath.Ext(info.Path)) {
				resStruct.Details = append(resStruct.Details, "Filetype not allowed: "+info.Name+" - "+filepath.Ext(info.Path))
				continue
			}

			quitChan := make(chan int)
			defer close(quitChan)
			doneChan := make(chan int)
			defer close(doneChan)

			cchan := utils.ScanFile(info, doneChan, quitChan, scannerSettings, pScanner)
			//END - Scan file from Disk with saving it locally

			// START - Scan file buffered data without saving it to disk
			/*quitChan := make(chan int)
			defer close(quitChan)

			cchan := utils.ScanBuffer(data, "", quitChan, scannerSettings, pScanner)*/
			// END - Scan file buffered data without saving it to disk

			if cresult := <-cchan; cresult.Err != nil {
				c.Err = cresult.Err
			} else {
				resStruct.ThreatResults = append(resStruct.ThreatResults, cresult)
			}

			// discard file after scanning (so disk won't fill)
			if err := os.Remove(info.Path); err != nil {
				log.Debug("Unable to delete file ", info.Path, " from disk. Error: ", err.Error())
			}
		}
		res := []byte(resStruct.ToJson())
		if dumpJsonToFile {
			scanLogger := utils.InitScanLogger(c.AccountId, projectId, appId, scannerId)
			logFile, lfErr := scanLogger.OpenLogFile()
			if lfErr != nil {
				return
			}
			defer logFile.Close()
			scanLogger.WriteToFile(logFile, resStruct, scanStartTimestamp)
		}
		w.Write(res)
	}
}

func getScanner(c *Context, w http.ResponseWriter, r *http.Request) {
	params := mux.Vars(r)
	// return if provided data for scanner is not valid
	if scanId, valid := validScannerData(c, params); valid {
		result := make(map[string]interface{})
		result["scanner_id"] = scanId
		w.Write([]byte(model.StringInterfaceToJson(result)))
	} else {
		c.SetInvalidParam("getScanner", "params")
	}

}

func validScannerData(c *Context, params map[string]string) (string, bool) {
	projectId := params["project_id"]
	appId := params["app_id"]
	scannerId := params["scanner_id"]

	if len(projectId) != model.PROJECT_ID_LENGTH {
		return scannerId, false
	}

	if len(appId) != model.APP_ID_LENGTH {
		return scannerId, false
	}

	// Check that app exists and belongs to this Account and Project
	if !IsValidApp(appId, projectId, c.AccountId) {
		return scannerId, false
	}

	if len(scannerId) != 0 && len(scannerId) != model.SCANNER_ID_LENGTH {
		return scannerId, false
	}

	return scannerId, true
}

func getAppDataForScanner(c *Context, projectId string, appId string) (license string, settingsWrapper *model.SettingsWrapper, valid bool) {
	cchan := Srv.Store.App().GetByProjectId(c.AccountId, projectId, appId)

	if cresult := <-cchan; cresult.Err != nil {
		c.Err = cresult.Err
		return license, settingsWrapper, false
	} else {
		app := cresult.Data.(*model.App)

		if protectionProfileResult := <-Srv.Store.ProtectionProfile().GetForApp(app); protectionProfileResult.Err != nil {
			c.Err = protectionProfileResult.Err
			return license, settingsWrapper, false
		} else {

			protectionProfile := &model.ProtectionProfile{}
			protectionProfile = protectionProfileResult.Data.(*model.ProtectionProfile)

			protectionProfileSettings := protectionProfile.ProfileSettings
			if len(protectionProfileSettings) == 0 {
				profileSettingsResult := <-Srv.Store.ProtectionProfile().BuildSettings(protectionProfile)
				if profileSettingsResult.Err != nil {
					c.Err = profileSettingsResult.Err
					return license, settingsWrapper, false
				}

				protectionProfileSettings = profileSettingsResult.Data.(string)
			}

			settingsWrapper = model.SettingsWrapperFromJson(strings.NewReader(protectionProfileSettings))

			if licenseResult := <-Srv.Store.License().GetByAccount(c.AccountId); licenseResult.Err != nil {
				c.Err = licenseResult.Err
				return license, settingsWrapper, false
			} else {

				var currentPlan model.Plan
				licenseRecord := licenseResult.Data.(*model.LicenseRecord)

				for _, plan := range *utils.Plans {
					if plan.Name == licenseRecord.PlanType {
						currentPlan = plan
					}
				}

				expirationDate := licenseRecord.ComputeExpirationDate()

				if currentPlan.Name == "" {
					c.Err = model.NewAppError("getApp", "api.app.get_app.invalid_license", map[string]interface{}{"AppId": appId, "AccountId": c.AccountId}, "")
					return license, settingsWrapper, false
				}

				for key := range currentPlan.ModulesInfo {
					currentPlan.ModulesInfo[key].ExpirationDate = expirationDate
				}

				licenseDetails := model.LicenseInfo{c.Account.CompanyName, c.Account.ApiId, strconv.FormatInt(licenseRecord.CreateAt, 10)}

				licenseObj := model.License{app.Id, licenseDetails, currentPlan.ModulesInfo}
				licenseWrapper := &model.LicenseWrapper{licenseObj}
				signature, sErr := model.SignLicense(licenseWrapper)
				if sErr != nil {
					c.Err = model.NewAppError("getApp", "api.app.get_app.invalid_license_signature", map[string]interface{}{"AppId": appId, "AccountId": c.AccountId}, "")
					return license, settingsWrapper, false
				}
				signatureHex := hex.EncodeToString(signature)

				/*
					license response is of form:
					//signature: 0123aabbcc
					{"license":{"app_id": "abcd", "modules_info": ...}}
				*/
				license = "// signature: " + signatureHex + "\n" + licenseWrapper.ToJson()

				if settingsWrapper == nil {
					c.Err = model.NewAppError("getApp", "api.app.get_app.invalid_settings_object", map[string]interface{}{"AppId": appId, "AccountId": c.AccountId}, "")
					return license, settingsWrapper, false
				}
				settings := settingsWrapper.SettingsObject
				settings = settings.ValidateSettings(&licenseObj)
				//settingsWrapper = &model.SettingsWrapper{settings}

				lsError := setLastSeen(c.AccountId, projectId, appId)
				if lsError != nil {
					c.Err = lsError
				}
			}
		}
	}

	return license, settingsWrapper, true
}