Add test to process large FLE file

flecmd/csv.go

@@ -27,34 +27,39 @@ import (
 var outputCsvFilename string
 var isOverwriteCsv bool
 
+var processCsvCommand = fleprocess.ProcessCsvCommand
+var csvCmd = csvCmdConstructor()
+
 // csvCmd is executed when choosing the csv option (load FLE file and generate csv file)
-var csvCmd = &cobra.Command{
+func csvCmdConstructor() *cobra.Command {
-	Use:   "csv [flags] inputFile [outputFile]",
+	return &cobra.Command{
-	Short: "Generates a SOTA .csv file based on a FLE type shorthand logfile.",
+		Use:   "csv [flags] inputFile [outputFile]",
-	// 	Long: `A longer description that spans multiple lines and likely contains examples
+		Short: "Generates a SOTA .csv file based on a FLE type shorthand logfile.",
-	// and usage of using your command. For example:
+		// 	Long: `A longer description that spans multiple lines and likely contains examples
+		// and usage of using your command. For example:
 
-	RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
-		//if args is empty, throw an error (Cobra will display the )
+			//if args is empty, throw an error (Cobra will display the )
-		if len(args) == 0 {
+			if len(args) == 0 {
-			//TODO: fix this ugly statement (because I am lazy)
+				//TODO: fix this ugly statement (because I am lazy)
-			return fmt.Errorf("Missing input file %s", "")
+				return fmt.Errorf("Missing input file %s", "")
-		}
+			}
-		inputFilename = args[0]
+			inputFilename = args[0]
-		if len(args) == 2 {
+			if len(args) == 2 {
-			outputCsvFilename = args[1]
+				outputCsvFilename = args[1]
-		}
+			}
-		if len(args) > 2 {
+			if len(args) > 2 {
-			return fmt.Errorf("Too many arguments.%s", "")
+				return fmt.Errorf("Too many arguments.%s", "")
-		}
+			}
 
-		if err := fleprocess.ProcessCsvCommand(inputFilename, outputCsvFilename, isInterpolateTime, isOverwriteCsv); err != nil {
+			if err := fleprocess.ProcessCsvCommand(inputFilename, outputCsvFilename, isInterpolateTime, isOverwriteCsv); err != nil {
-			fmt.Println("\nUnable to generate CSV file:")
+				fmt.Println("\nUnable to generate CSV file:")
-			fmt.Println(err)
+				fmt.Println(err)
-			os.Exit(1)
+				os.Exit(1)
-		}
+			}
-		return nil
+			return nil
-	},
+		},
+	}
 }
 
 func init() {

flecmd/load.go

@@ -19,33 +19,33 @@ limitations under the License.
 import (
 	"FLEcli/fleprocess"
 	"fmt"
-
 	"github.com/spf13/cobra"
 )
 
+var processLoadFile = fleprocess.LoadFile
+var loadCmd = loadCmdConstructor()
+
 // loadCmd represents the load command
-var loadCmd = &cobra.Command{
+func loadCmdConstructor() *cobra.Command {
-	Use:   "load [flags] inputFile",
+	return &cobra.Command{
-	Short: "Loads and validates a FLE type shorthand logfile",
+		Use:   "load [flags] inputFile",
-	// 	Long: `A longer description that spans multiple lines and likely contains examples
+		Short: "Loads and validates a FLE type shorthand logfile",
-	// and usage of using your command. For example:
+
-
+		RunE: func(cmd *cobra.Command, args []string) error {
-	// Cobra is a CLI library for Go that empowers applications.
+			//if args is empty, throw an error
-	// This application is a tool to generate the needed files
+			if len(args) < 1 {
-	// to quickly create a Cobra application.`,
+				//FIXME: Doesn't work as expected
-	RunE: func(cmd *cobra.Command, args []string) error {
+				return fmt.Errorf("Missing input file %s", "")
-		//if args is empty, throw an error
+			}
-		if len(args) == 0 {
+			if len(args) > 1 {
-			//TODO: fix this ugly statement (because I am lazy)
+				return fmt.Errorf("Too many arguments.%s", "")
-			return fmt.Errorf("Missing input file %s", "")
+			}
-		}
+			inputFilename = args[0]
-		if len(args) > 1 {
+			//FIXME: we should return the result of the call
-			return fmt.Errorf("Too many arguments.%s", "")
+			processLoadFile(inputFilename, isInterpolateTime)
-		}
+			return nil
-		inputFilename = args[0]
+		},
-		fleprocess.LoadFile(inputFilename, isInterpolateTime)
+	}
-		return nil
-	},
 }
 
 func init() {

flecmd/load_test.go

@@ -0,0 +1,111 @@
+package flecmd
+
+/*
+Copyright © 2020 Jean-Marc Meessen, ON4KJM <on4kjm@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import (
+	"FLEcli/fleprocess"
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+
+	"io/ioutil"
+	"testing"
+)
+
+func Test_ExecuteCommand_help(t *testing.T) {
+	cmd := loadCmdConstructor()
+	b := bytes.NewBufferString("")
+	cmd.SetOut(b)
+	cmd.SetArgs([]string{"--help"})
+	cmd.Execute()
+	out, err := ioutil.ReadAll(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedOutput := "Loads and validates a FLE type shorthand logfile\n\nUsage:\n  load [flags] inputFile\n\nFlags:\n  -h, --help   help for load\n"
+	if string(out) != expectedOutput {
+		t.Fatalf("expected \"%s\" got \"%s\"", expectedOutput, string(out))
+	}
+}
+
+func Test_ExecuteCommand_noArgs(t *testing.T) {
+	cmd := loadCmdConstructor()
+	b := bytes.NewBufferString("")
+	cmd.SetOut(b)
+	//cmd.SetArgs([]string{""})
+	cmd.Execute()
+	out, err := ioutil.ReadAll(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//FIXME: doesn't work as espected
+	expectedOutputStart := "Error: Missing input file \nUsage:"
+	if !strings.HasPrefix(string(out), expectedOutputStart) {
+		t.Fatalf("expected to start with \"%s\" got \"%s\"", expectedOutputStart, string(out))
+	}
+}
+
+func Test_ExecuteCommand_toManyArgs(t *testing.T) {
+	cmd := loadCmdConstructor()
+	b := bytes.NewBufferString("")
+	cmd.SetOut(b)
+	cmd.SetArgs([]string{"blaah", "blaah", "blaah"})
+	cmd.Execute()
+	out, err := ioutil.ReadAll(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expectedOutputStart := "Error: Too many arguments.\nUsage"
+	if !strings.HasPrefix(string(out), expectedOutputStart) {
+		t.Fatalf("expected to start with \"%s\" got \"%s\"", expectedOutputStart, string(out))
+	}
+}
+
+func Test_ExecuteCommand_happyCase(t *testing.T) {
+	processLoadFile = mockLoadFile
+
+	//Capture output
+	rescueStdout := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+
+	cmd := loadCmdConstructor()
+
+	cmd.SetArgs([]string{"data.txt"})
+	cmdErr := cmd.Execute()
+
+	//Close the capture and get the data
+	w.Close()
+	out, _ := ioutil.ReadAll(r)
+	os.Stdout = rescueStdout
+
+	if cmdErr != nil {
+		t.Fatalf("Unexpected error executing command: %s", cmdErr)
+	}
+
+	if string(out) != "fileLoad via mock" {
+		t.Fatalf("Expected \"fileLoad via mock\". Got \"%s\"", string(out))
+	}
+}
+
+func mockLoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []fleprocess.LogLine, isProcessedOK bool) {
+	fmt.Print("fileLoad via mock")
+	return nil, true
+}

fleprocess/adif_process_test.go

@@ -1,5 +1,21 @@
 package fleprocess
 
+/*
+Copyright © 2020 Jean-Marc Meessen, ON4KJM <on4kjm@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import (
 	"fmt"
 	"testing"

fleprocess/adif_write.go

@@ -70,7 +70,7 @@ func buildAdif(fullLog []LogLine, isWWFF bool, isSOTA bool) (adifList []string)
 			adifLine.WriteString(adifElement("MY_SIG_INFO", logLine.MyWWFF))
 			if logLine.WWFF != "" {
 				adifLine.WriteString(adifElement("SIG", "WWFF"))
-				adifLine.WriteString(adifElement("SIG_INFO",logLine.WWFF))
+				adifLine.WriteString(adifElement("SIG_INFO", logLine.WWFF))
 			}
 		}
 		if isSOTA {

fleprocess/csv_process.go

@@ -111,20 +111,19 @@ func validateDataForSotaCsv(loadedLogFile []LogLine) error {
 			}
 			errorsBuffer.WriteString(fmt.Sprintf("missing QSO time %s", errorLocation))
 		}
-		//FIXME: if isNoMySota and MySota defined means that it was defined later in the log file 
+		//FIXME: if isNoMySota and MySota defined means that it was defined later in the log file
-		if (isNoMySota && loadedLogFile[i].MySOTA != "") {
+		if isNoMySota && loadedLogFile[i].MySOTA != "" {
 			if errorsBuffer.String() != "" {
 				errorsBuffer.WriteString(fmt.Sprintf(", "))
 			}
-			errorsBuffer.WriteString(fmt.Sprintf("encountered an unexpexted MySota reference while processing what should be a chaser log %s", errorLocation))	
+			errorsBuffer.WriteString(fmt.Sprintf("encountered an unexpexted MySota reference while processing what should be a chaser log %s", errorLocation))
 		}
 
-
+		if isNoMySota && loadedLogFile[i].SOTA == "" {
-		if (isNoMySota && loadedLogFile[i].SOTA == "") {
 			if errorsBuffer.String() != "" {
 				errorsBuffer.WriteString(fmt.Sprintf(", "))
 			}
-			errorsBuffer.WriteString(fmt.Sprintf("missing SOTA reference while attempting to process chaser log %s", errorLocation))	
+			errorsBuffer.WriteString(fmt.Sprintf("missing SOTA reference while attempting to process chaser log %s", errorLocation))
 		}
 	}
 	if errorsBuffer.String() != "" {

fleprocess/inferTime.go

@@ -54,11 +54,10 @@ func (tb *InferTimeBlock) String() string {
 //finalizeTimeGap makes the necessary checks and computation
 func (tb *InferTimeBlock) finalizeTimeGap() error {
 
-	if err :=tb.validateTimeGap(); err != nil {
+	if err := tb.validateTimeGap(); err != nil {
 		return err
 	}
 
-
 	//Compute the gap
 	diff := tb.nextValidTime.Sub(tb.lastRecordedTime)
 	tb.deltatime = time.Duration(diff / time.Duration(tb.noTimeCount+1))
@@ -74,7 +73,7 @@ func (tb *InferTimeBlock) finalizeTimeGap() error {
 }
 
 //validateTimeGap checks some important assumptions
-func (tb *InferTimeBlock) validateTimeGap() error{
+func (tb *InferTimeBlock) validateTimeGap() error {
 	//Check that lastRecordedTime and nextValidTime are not null
 	if tb.lastRecordedTime.IsZero() {
 		return errors.New("Gap start time is empty")

fleprocess/load_file_test.go

@@ -395,8 +395,6 @@ func TestLoadFile_redefining_myCall_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "myCall on4do")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -435,8 +433,6 @@ func TestLoadFile_redefining_myWWFF_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "myWWFF onff-0001")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -475,8 +471,6 @@ func TestLoadFile_redefining_mySOTA_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "mySota on/on-111")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -516,8 +510,6 @@ func TestLoadFile_redefining_myGRID_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "myGrid ZZ99")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -540,8 +532,6 @@ func TestLoadFile_redefining_myGRID_must_fail(t *testing.T) {
 	os.Remove(temporaryDataFileName)
 }
 
-
-
 func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
 
 	//Given
@@ -558,8 +548,6 @@ func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "operator blahh")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -582,7 +570,6 @@ func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
 	os.Remove(temporaryDataFileName)
 }
 
-
 func TestLoadFile_redefining_nickname_must_fail(t *testing.T) {
 
 	//Given
@@ -599,8 +586,6 @@ func TestLoadFile_redefining_nickname_must_fail(t *testing.T) {
 	dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
 	dataArray = append(dataArray, "nickname blaaahh")
 
-
-
 	temporaryDataFileName := createTestFile(dataArray)
 
 	//When
@@ -823,7 +808,7 @@ func TestLoadFile_InferTime_missingStartTime(t *testing.T) {
 	if loadedLogFile[0].MyWWFF != expectedValue {
 		t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue)
 	}
-	
+
 	expectedValue = "IK5ZVE"
 	if loadedLogFile[0].Call != expectedValue {
 		t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
@@ -896,7 +881,7 @@ func TestLoadFile_InferTime_missingEndTime(t *testing.T) {
 	if loadedLogFile[0].MyWWFF != expectedValue {
 		t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue)
 	}
-	
+
 	expectedValue = "IK5ZVE"
 	if loadedLogFile[0].Call != expectedValue {
 		t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
@@ -972,7 +957,7 @@ func TestLoadFile_2_QSO_same_time(t *testing.T) {
 	if loadedLogFile[0].MyWWFF != expectedValue {
 		t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue)
 	}
-	
+
 	expectedValue = "IK5ZVE"
 	if loadedLogFile[0].Call != expectedValue {
 		t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
@@ -1021,8 +1006,6 @@ func TestLoadFile_2_QSO_same_time(t *testing.T) {
 	os.Remove(temporaryDataFileName)
 }
 
-
-
 func TestLoadFile_wrongData(t *testing.T) {
 
 	//Given

fleprocess/parse_line.go

@@ -154,7 +154,6 @@ func ParseLine(inputStr string, previousLine LogLine) (logLine LogLine, errorMsg
 		//Scan the + part
 		if regexpDayIncrementPattern.MatchString(element) {
 			increment := len(element)
-			fmt.Println(logLine.Date)
 			newDate, dateError := IncrementDate(logLine.Date, increment)
 			if dateError != "" {
 				errorMsg = errorMsg + fmt.Sprintf(dateError)

fleprocess/parse_line_test.go

@@ -57,7 +57,7 @@ func TestParseLine(t *testing.T) {
 			"Parse suspecious line",
 			args{inputStr: "1029 Sm/dl8mf 559 579", previousLine: LogLine{Time: "1200", Mode: "CW", ModeType: "CW"}},
 			LogLine{Time: "1029", ActualTime: "1029", Call: "SM/DL8MF", Mode: "CW", ModeType: "CW", RSTsent: "559", RSTrcvd: "579"}, "",
-		},		
+		},
 		{
 			"Parse partial time - 3",
 			args{inputStr: "4 g3noh", previousLine: LogLine{Time: "1200", Mode: "SSB"}},
@@ -178,6 +178,11 @@ func TestParseLine(t *testing.T) {
 			args{inputStr: "day ++ 1230 oe6cud/p ", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},
 			LogLine{Date: "2020-09-07", Call: "OE6CUD/P", Time: "1230", ActualTime: "1230", RSTsent: "59", RSTrcvd: "59", Mode: "FM", ModeType: "PHONE"}, "",
 		},
+		{
+			"date processing - day single line",
+			args{inputStr: "day ++", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},
+			LogLine{Date: "2020-09-07", RSTsent: "59", RSTrcvd: "59", Mode: "FM", ModeType: "PHONE"}, "",
+		},
 		{
 			"date processing - day (error) ",
 			args{inputStr: "day +++++++++++ 1230 oe6cud/p ", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},

fleprocess/validate_test.go

@@ -191,7 +191,7 @@ func TestValidateCall(t *testing.T) {
 			"Valid call from activation (case 2)",
 			args{sign: "Sm/dl8mf"},
 			"SM/DL8MF", "",
-		},		
+		},
 		{
 			"Valid prefix (issue #2)",
 			args{sign: "e7/z35m/p"},

test/bats-scripts/test.bats

@@ -4,29 +4,22 @@
 
 @test "Does the version show?" {
     output=$(docker run --rm on4kjm/flecli:latest version -d)
-    echo 'status:' $status
+#   [ "$status" -eq 0 ]
-    echo 'output:' $output
 }
 
 
 @test "Can a simple file be loaded?" {
     output=$(test/docker-FLEcli.sh load -i test/data/fle-1.txt)
-    echo 'status:' $status
-    echo 'output:' $output
 }
 
 
 @test "Can a more complex file be loaded?" {
     output=$(test/docker-FLEcli.sh load -i test/data/ON4KJM@ONFF-025920200524.txt)
-    echo 'status:' $status
-    echo 'output:' $output
 }
 
 @test "Is the generated SOTA csv equivalent to the canonical one?" {
     mkdir -p test/output/temp
     output=$(test/docker-FLEcli.sh csv -o -i test/FLE-sample/sota_wwff.txt test/output/temp/sota_wwff.csv)
-    echo 'status:' $status
-    echo 'output:' $output
     diff test/output/temp/sota_wwff.csv test/FLE-sample/sota_wwff.csv --strip-trailing-cr
 }
 
@@ -38,4 +31,8 @@
 @test "Processing a FLE file with parsing errors must fail!" {
     run test/docker-FLEcli.sh csv -o -i test/data/fle-5-wrong-call.txt
     [ "$status" -eq 1 ]
+}
+
+@test "Processing a big FLE file" {
+    run test/docker-FLEcli.sh csv -o -i test/data/fle-6-bigFile.txt test/output/temp/fle-6-bigFile.csv
 }

test/test.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e
-
-./build.sh
-
-
-
-echo "--------------------------"
-./FLEcli load -i test/data/fle-1.txt
-echo "--------------------------"
-./FLEcli load -i test/data/ON4KJM@ONFF-025920200524.txt
-echo "--------------------------"
-mkdir -p test/output/temp
-./FLEcli csv -o -i test/FLE-sample/sota_wwff.txt test/output/temp/sota_wwff.csv
-diff test/output/temp/sota_wwff.csv test/FLE-sample/sota_wwff.csv --strip-trailing-cr && echo "no difference" || echo "differences!"
-echo "--------------------------"
-./FLEcli -i test/FLE-sample/sota_wwff.txt adif -o=test/output/temp/sota_wwff.adif --interpolate --overwrite --wwff --sota