Add test to process large FLE file

flecmd/csv.go

@@ -27,8 +27,12 @@ import (
 var outputCsvFilename string
 var isOverwriteCsv bool
 
+var processCsvCommand = fleprocess.ProcessCsvCommand
+var csvCmd = csvCmdConstructor()
+
 // csvCmd is executed when choosing the csv option (load FLE file and generate csv file)
-var csvCmd = &cobra.Command{
+func csvCmdConstructor() *cobra.Command {
+	return &cobra.Command{
 		Use:   "csv [flags] inputFile [outputFile]",
 		Short: "Generates a SOTA .csv file based on a FLE type shorthand logfile.",
 		// 	Long: `A longer description that spans multiple lines and likely contains examples
@@ -55,6 +59,7 @@ var csvCmd = &cobra.Command{
 			}
 			return nil
 		},
+	}
 }
 
 func init() {

flecmd/load.go

@@ -19,33 +19,33 @@ limitations under the License.
 import (
 	"FLEcli/fleprocess"
 	"fmt"
-
 	"github.com/spf13/cobra"
 )
 
+var processLoadFile = fleprocess.LoadFile
+var loadCmd = loadCmdConstructor()
+
 // loadCmd represents the load command
-var loadCmd = &cobra.Command{
+func loadCmdConstructor() *cobra.Command {
+	return &cobra.Command{
 		Use:   "load [flags] inputFile",
 		Short: "Loads and validates a FLE type shorthand logfile",
-	// 	Long: `A longer description that spans multiple lines and likely contains examples
-	// and usage of using your command. For example:
 
-	// Cobra is a CLI library for Go that empowers applications.
-	// This application is a tool to generate the needed files
-	// to quickly create a Cobra application.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			//if args is empty, throw an error
-		if len(args) == 0 {
+			if len(args) < 1 {
-			//TODO: fix this ugly statement (because I am lazy)
+				//FIXME: Doesn't work as expected
 				return fmt.Errorf("Missing input file %s", "")
 			}
 			if len(args) > 1 {
 				return fmt.Errorf("Too many arguments.%s", "")
 			}
 			inputFilename = args[0]
-		fleprocess.LoadFile(inputFilename, isInterpolateTime)
+			//FIXME: we should return the result of the call
+			processLoadFile(inputFilename, isInterpolateTime)
 			return nil
 		},
+	}
 }
 
 func init() {

flecmd/load_test.go

@@ -0,0 +1,111 @@
+package flecmd
+
+/*
+Copyright © 2020 Jean-Marc Meessen, ON4KJM <on4kjm@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import (
+	"FLEcli/fleprocess"
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+
+	"io/ioutil"
+	"testing"
+)
+
+func Test_ExecuteCommand_help(t *testing.T) {
+	cmd := loadCmdConstructor()
+	b := bytes.NewBufferString("")
+	cmd.SetOut(b)
+	cmd.SetArgs([]string{"--help"})
+	cmd.Execute()
+	out, err := ioutil.ReadAll(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedOutput := "Loads and validates a FLE type shorthand logfile\n\nUsage:\n  load [flags] inputFile\n\nFlags:\n  -h, --help   help for load\n"
+	if string(out) != expectedOutput {
+		t.Fatalf("expected \"%s\" got \"%s\"", expectedOutput, string(out))
+	}
+}
+
+func Test_ExecuteCommand_noArgs(t *testing.T) {
+	cmd := loadCmdConstructor()
+	b := bytes.NewBufferString("")
+	cmd.SetOut(b)
+	//cmd.SetArgs([]string{""})
+	cmd.Execute()
+	out, err := ioutil.ReadAll(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//FIXME: doesn't work as espected
+	expectedOutputStart := "Error: Missing input file \nUsage:"
+	if !strings.HasPrefix(string(out), expectedOutputStart) {
+		t.Fatalf("expected to start with \"%s\" got \"%s\"", expectedOutputStart, string(out))
+	}
+}
+
+func Test_ExecuteCommand_toManyArgs(t *testing.T) {
+	cmd := loadCmdConstructor()
+	b := bytes.NewBufferString("")
+	cmd.SetOut(b)
+	cmd.SetArgs([]string{"blaah", "blaah", "blaah"})
+	cmd.Execute()
+	out, err := ioutil.ReadAll(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedOutputStart := "Error: Too many arguments.\nUsage"
+	if !strings.HasPrefix(string(out), expectedOutputStart) {
+		t.Fatalf("expected to start with \"%s\" got \"%s\"", expectedOutputStart, string(out))
+	}
+}
+
+func Test_ExecuteCommand_happyCase(t *testing.T) {
+	processLoadFile = mockLoadFile
+
+	//Capture output
+	rescueStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+
+	cmd := loadCmdConstructor()
+
+	cmd.SetArgs([]string{"data.txt"})
+	cmdErr := cmd.Execute()
+
+	//Close the capture and get the data
+	w.Close()
+	out, _ := ioutil.ReadAll(r)
+	os.Stdout = rescueStdout
+
+	if cmdErr != nil {
+		t.Fatalf("Unexpected error executing command: %s", cmdErr)
+	}
+
+	if string(out) != "fileLoad via mock" {
+		t.Fatalf("Expected \"fileLoad via mock\". Got \"%s\"", string(out))
+	}
+}
+
+func mockLoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []fleprocess.LogLine, isProcessedOK bool) {
+	fmt.Print("fileLoad via mock")
+	return nil, true
+}

fleprocess/adif_process_test.go

@@ -1,5 +1,21 @@
 package fleprocess
 
+/*
+Copyright © 2020 Jean-Marc Meessen, ON4KJM <on4kjm@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import (
 	"fmt"
 	"testing"

fleprocess/adif_write.go

@@ -70,7 +70,7 @@ func buildAdif(fullLog []LogLine, isWWFF bool, isSOTA bool) (adifList []string)
 			adifLine.WriteString(adifElement("MY_SIG_INFO", logLine.MyWWFF))
 			if logLine.WWFF != "" {
 				adifLine.WriteString(adifElement("SIG", "WWFF"))
-				adifLine.WriteString(adifElement("SIG_INFO",logLine.WWFF))
+				adifLine.WriteString(adifElement("SIG_INFO", logLine.WWFF))
 			}
 		}
 		if isSOTA {

fleprocess/csv_process.go

@@ -112,15 +112,14 @@ func validateDataForSotaCsv(loadedLogFile []LogLine) error {
 			errorsBuffer.WriteString(fmt.Sprintf("missing QSO time %s", errorLocation))
 		}
 		//FIXME: if isNoMySota and MySota defined means that it was defined later in the log file
-		if (isNoMySota && loadedLogFile[i].MySOTA != "") {
+		if isNoMySota && loadedLogFile[i].MySOTA != "" {
 			if errorsBuffer.String() != "" {
 				errorsBuffer.WriteString(fmt.Sprintf(", "))
 			}
 			errorsBuffer.WriteString(fmt.Sprintf("encountered an unexpexted MySota reference while processing what should be a chaser log %s", errorLocation))
 		}
 
-
+		if isNoMySota && loadedLogFile[i].SOTA == "" {
-		if (isNoMySota && loadedLogFile[i].SOTA == "") {
 			if errorsBuffer.String() != "" {
 				errorsBuffer.WriteString(fmt.Sprintf(", "))
 			}

fleprocess/inferTime.go

@@ -54,11 +54,10 @@ func (tb *InferTimeBlock) String() string {
 //finalizeTimeGap makes the necessary checks and computation
 func (tb *InferTimeBlock) finalizeTimeGap() error {
 
-	if err :=tb.validateTimeGap(); err != nil {
+	if err := tb.validateTimeGap(); err != nil {
 		return err
 	}
 
-
 	//Compute the gap
 	diff := tb.nextValidTime.Sub(tb.lastRecordedTime)
 	tb.deltatime = time.Duration(diff / time.Duration(tb.noTimeCount+1))
@@ -74,7 +73,7 @@ func (tb *InferTimeBlock) finalizeTimeGap() error {
 }
 
 //validateTimeGap checks some important assumptions
-func (tb *InferTimeBlock) validateTimeGap() error{
+func (tb *InferTimeBlock) validateTimeGap() error {
 	//Check that lastRecordedTime and nextValidTime are not null
 	if tb.lastRecordedTime.IsZero() {
 		return errors.New("Gap start time is empty")

fleprocess/load_file_test.go

@@ -395,8 +395,6 @@ func TestLoadFile_redefining_myCall_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "myCall on4do")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -435,8 +433,6 @@ func TestLoadFile_redefining_myWWFF_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "myWWFF onff-0001")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -475,8 +471,6 @@ func TestLoadFile_redefining_mySOTA_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "mySota on/on-111")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -516,8 +510,6 @@ func TestLoadFile_redefining_myGRID_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "myGrid ZZ99")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -540,8 +532,6 @@ func TestLoadFile_redefining_myGRID_must_fail(t *testing.T) {
 	os.Remove(temporaryDataFileName)
 }
 
-
-
 func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
 
 	//Given
@@ -558,8 +548,6 @@ func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "operator blahh")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -582,7 +570,6 @@ func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
 	os.Remove(temporaryDataFileName)
 }
 
-
 func TestLoadFile_redefining_nickname_must_fail(t *testing.T) {
 
 	//Given
@@ -599,8 +586,6 @@ func TestLoadFile_redefining_nickname_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "nickname blaaahh")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -1021,8 +1006,6 @@ func TestLoadFile_2_QSO_same_time(t *testing.T) {
 	os.Remove(temporaryDataFileName)
 }
 
-
-
 func TestLoadFile_wrongData(t *testing.T) {
 
 	//Given

fleprocess/parse_line.go

@@ -154,7 +154,6 @@ func ParseLine(inputStr string, previousLine LogLine) (logLine LogLine, errorMsg
 		//Scan the + part
 		if regexpDayIncrementPattern.MatchString(element) {
 			increment := len(element)
-			fmt.Println(logLine.Date)
 			newDate, dateError := IncrementDate(logLine.Date, increment)
 			if dateError != "" {
 				errorMsg = errorMsg + fmt.Sprintf(dateError)

fleprocess/parse_line_test.go

@@ -178,6 +178,11 @@ func TestParseLine(t *testing.T) {
 			args{inputStr: "day ++ 1230 oe6cud/p ", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},
 			LogLine{Date: "2020-09-07", Call: "OE6CUD/P", Time: "1230", ActualTime: "1230", RSTsent: "59", RSTrcvd: "59", Mode: "FM", ModeType: "PHONE"}, "",
 		},
+		{
+			"date processing - day single line",
+			args{inputStr: "day ++", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},
+			LogLine{Date: "2020-09-07", RSTsent: "59", RSTrcvd: "59", Mode: "FM", ModeType: "PHONE"}, "",
+		},
 		{
 			"date processing - day (error) ",
 			args{inputStr: "day +++++++++++ 1230 oe6cud/p ", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},

test/bats-scripts/test.bats

@@ -4,29 +4,22 @@
 
 @test "Does the version show?" {
     output=$(docker run --rm on4kjm/flecli:latest version -d)
-    echo 'status:' $status
+#   [ "$status" -eq 0 ]
-    echo 'output:' $output
 }
 
 
 @test "Can a simple file be loaded?" {
     output=$(test/docker-FLEcli.sh load -i test/data/fle-1.txt)
-    echo 'status:' $status
-    echo 'output:' $output
 }
 
 
 @test "Can a more complex file be loaded?" {
     output=$(test/docker-FLEcli.sh load -i test/data/ON4KJM@ONFF-025920200524.txt)
-    echo 'status:' $status
-    echo 'output:' $output
 }
 
 @test "Is the generated SOTA csv equivalent to the canonical one?" {
     mkdir -p test/output/temp
     output=$(test/docker-FLEcli.sh csv -o -i test/FLE-sample/sota_wwff.txt test/output/temp/sota_wwff.csv)
-    echo 'status:' $status
-    echo 'output:' $output
     diff test/output/temp/sota_wwff.csv test/FLE-sample/sota_wwff.csv --strip-trailing-cr
 }
 
@@ -39,3 +32,7 @@
     run test/docker-FLEcli.sh csv -o -i test/data/fle-5-wrong-call.txt
     [ "$status" -eq 1 ]
 }
+
+@test "Processing a big FLE file" {
+    run test/docker-FLEcli.sh csv -o -i test/data/fle-6-bigFile.txt test/output/temp/fle-6-bigFile.csv
+}

test/test.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e
-
-./build.sh
-
-
-
-echo "--------------------------"
-./FLEcli load -i test/data/fle-1.txt
-echo "--------------------------"
-./FLEcli load -i test/data/ON4KJM@ONFF-025920200524.txt
-echo "--------------------------"
-mkdir -p test/output/temp
-./FLEcli csv -o -i test/FLE-sample/sota_wwff.txt test/output/temp/sota_wwff.csv
-diff test/output/temp/sota_wwff.csv test/FLE-sample/sota_wwff.csv --strip-trailing-cr && echo "no difference" || echo "differences!"
-echo "--------------------------"
-./FLEcli -i test/FLE-sample/sota_wwff.txt adif -o=test/output/temp/sota_wwff.adif --interpolate --overwrite --wwff --sota