Add test to process large FLE file

pull/75/head
Jean-Marc MEESSEN 4 years ago committed by GitHub
parent 8d03018796
commit ce146e3407
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -27,34 +27,39 @@ import (
var outputCsvFilename string
var isOverwriteCsv bool
var processCsvCommand = fleprocess.ProcessCsvCommand
var csvCmd = csvCmdConstructor()
// csvCmd is executed when choosing the csv option (load FLE file and generate csv file)
var csvCmd = &cobra.Command{
Use: "csv [flags] inputFile [outputFile]",
Short: "Generates a SOTA .csv file based on a FLE type shorthand logfile.",
// Long: `A longer description that spans multiple lines and likely contains examples
// and usage of using your command. For example:
func csvCmdConstructor() *cobra.Command {
return &cobra.Command{
Use: "csv [flags] inputFile [outputFile]",
Short: "Generates a SOTA .csv file based on a FLE type shorthand logfile.",
// Long: `A longer description that spans multiple lines and likely contains examples
// and usage of using your command. For example:
RunE: func(cmd *cobra.Command, args []string) error {
//if args is empty, throw an error (Cobra will display the )
if len(args) == 0 {
//TODO: fix this ugly statement (because I am lazy)
return fmt.Errorf("Missing input file %s", "")
}
inputFilename = args[0]
if len(args) == 2 {
outputCsvFilename = args[1]
}
if len(args) > 2 {
return fmt.Errorf("Too many arguments.%s", "")
}
RunE: func(cmd *cobra.Command, args []string) error {
//if args is empty, throw an error (Cobra will display the )
if len(args) == 0 {
//TODO: fix this ugly statement (because I am lazy)
return fmt.Errorf("Missing input file %s", "")
}
inputFilename = args[0]
if len(args) == 2 {
outputCsvFilename = args[1]
}
if len(args) > 2 {
return fmt.Errorf("Too many arguments.%s", "")
}
if err := fleprocess.ProcessCsvCommand(inputFilename, outputCsvFilename, isInterpolateTime, isOverwriteCsv); err != nil {
fmt.Println("\nUnable to generate CSV file:")
fmt.Println(err)
os.Exit(1)
}
return nil
},
if err := fleprocess.ProcessCsvCommand(inputFilename, outputCsvFilename, isInterpolateTime, isOverwriteCsv); err != nil {
fmt.Println("\nUnable to generate CSV file:")
fmt.Println(err)
os.Exit(1)
}
return nil
},
}
}
func init() {

@ -19,33 +19,33 @@ limitations under the License.
import (
"FLEcli/fleprocess"
"fmt"
"github.com/spf13/cobra"
)
var processLoadFile = fleprocess.LoadFile
var loadCmd = loadCmdConstructor()
// loadCmd represents the load command
var loadCmd = &cobra.Command{
Use: "load [flags] inputFile",
Short: "Loads and validates a FLE type shorthand logfile",
// Long: `A longer description that spans multiple lines and likely contains examples
// and usage of using your command. For example:
// Cobra is a CLI library for Go that empowers applications.
// This application is a tool to generate the needed files
// to quickly create a Cobra application.`,
RunE: func(cmd *cobra.Command, args []string) error {
//if args is empty, throw an error
if len(args) == 0 {
//TODO: fix this ugly statement (because I am lazy)
return fmt.Errorf("Missing input file %s", "")
}
if len(args) > 1 {
return fmt.Errorf("Too many arguments.%s", "")
}
inputFilename = args[0]
fleprocess.LoadFile(inputFilename, isInterpolateTime)
return nil
},
func loadCmdConstructor() *cobra.Command {
return &cobra.Command{
Use: "load [flags] inputFile",
Short: "Loads and validates a FLE type shorthand logfile",
RunE: func(cmd *cobra.Command, args []string) error {
//if args is empty, throw an error
if len(args) < 1 {
//FIXME: Doesn't work as expected
return fmt.Errorf("Missing input file %s", "")
}
if len(args) > 1 {
return fmt.Errorf("Too many arguments.%s", "")
}
inputFilename = args[0]
//FIXME: we should return the result of the call
processLoadFile(inputFilename, isInterpolateTime)
return nil
},
}
}
func init() {

@ -0,0 +1,111 @@
package flecmd
/*
Copyright © 2020 Jean-Marc Meessen, ON4KJM <on4kjm@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import (
"FLEcli/fleprocess"
"bytes"
"fmt"
"os"
"strings"
"io/ioutil"
"testing"
)
func Test_ExecuteCommand_help(t *testing.T) {
cmd := loadCmdConstructor()
b := bytes.NewBufferString("")
cmd.SetOut(b)
cmd.SetArgs([]string{"--help"})
cmd.Execute()
out, err := ioutil.ReadAll(b)
if err != nil {
t.Fatal(err)
}
expectedOutput := "Loads and validates a FLE type shorthand logfile\n\nUsage:\n load [flags] inputFile\n\nFlags:\n -h, --help help for load\n"
if string(out) != expectedOutput {
t.Fatalf("expected \"%s\" got \"%s\"", expectedOutput, string(out))
}
}
func Test_ExecuteCommand_noArgs(t *testing.T) {
cmd := loadCmdConstructor()
b := bytes.NewBufferString("")
cmd.SetOut(b)
//cmd.SetArgs([]string{""})
cmd.Execute()
out, err := ioutil.ReadAll(b)
if err != nil {
t.Fatal(err)
}
//FIXME: doesn't work as espected
expectedOutputStart := "Error: Missing input file \nUsage:"
if !strings.HasPrefix(string(out), expectedOutputStart) {
t.Fatalf("expected to start with \"%s\" got \"%s\"", expectedOutputStart, string(out))
}
}
func Test_ExecuteCommand_toManyArgs(t *testing.T) {
cmd := loadCmdConstructor()
b := bytes.NewBufferString("")
cmd.SetOut(b)
cmd.SetArgs([]string{"blaah", "blaah", "blaah"})
cmd.Execute()
out, err := ioutil.ReadAll(b)
if err != nil {
t.Fatal(err)
}
expectedOutputStart := "Error: Too many arguments.\nUsage"
if !strings.HasPrefix(string(out), expectedOutputStart) {
t.Fatalf("expected to start with \"%s\" got \"%s\"", expectedOutputStart, string(out))
}
}
func Test_ExecuteCommand_happyCase(t *testing.T) {
processLoadFile = mockLoadFile
//Capture output
rescueStdout := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
cmd := loadCmdConstructor()
cmd.SetArgs([]string{"data.txt"})
cmdErr := cmd.Execute()
//Close the capture and get the data
w.Close()
out, _ := ioutil.ReadAll(r)
os.Stdout = rescueStdout
if cmdErr != nil {
t.Fatalf("Unexpected error executing command: %s", cmdErr)
}
if string(out) != "fileLoad via mock" {
t.Fatalf("Expected \"fileLoad via mock\". Got \"%s\"", string(out))
}
}
func mockLoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []fleprocess.LogLine, isProcessedOK bool) {
fmt.Print("fileLoad via mock")
return nil, true
}

@ -1,5 +1,21 @@
package fleprocess
/*
Copyright © 2020 Jean-Marc Meessen, ON4KJM <on4kjm@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import (
"fmt"
"testing"

@ -70,7 +70,7 @@ func buildAdif(fullLog []LogLine, isWWFF bool, isSOTA bool) (adifList []string)
adifLine.WriteString(adifElement("MY_SIG_INFO", logLine.MyWWFF))
if logLine.WWFF != "" {
adifLine.WriteString(adifElement("SIG", "WWFF"))
adifLine.WriteString(adifElement("SIG_INFO",logLine.WWFF))
adifLine.WriteString(adifElement("SIG_INFO", logLine.WWFF))
}
}
if isSOTA {

@ -112,15 +112,14 @@ func validateDataForSotaCsv(loadedLogFile []LogLine) error {
errorsBuffer.WriteString(fmt.Sprintf("missing QSO time %s", errorLocation))
}
//FIXME: if isNoMySota and MySota defined means that it was defined later in the log file
if (isNoMySota && loadedLogFile[i].MySOTA != "") {
if isNoMySota && loadedLogFile[i].MySOTA != "" {
if errorsBuffer.String() != "" {
errorsBuffer.WriteString(fmt.Sprintf(", "))
}
errorsBuffer.WriteString(fmt.Sprintf("encountered an unexpexted MySota reference while processing what should be a chaser log %s", errorLocation))
}
if (isNoMySota && loadedLogFile[i].SOTA == "") {
if isNoMySota && loadedLogFile[i].SOTA == "" {
if errorsBuffer.String() != "" {
errorsBuffer.WriteString(fmt.Sprintf(", "))
}

@ -54,11 +54,10 @@ func (tb *InferTimeBlock) String() string {
//finalizeTimeGap makes the necessary checks and computation
func (tb *InferTimeBlock) finalizeTimeGap() error {
if err :=tb.validateTimeGap(); err != nil {
if err := tb.validateTimeGap(); err != nil {
return err
}
//Compute the gap
diff := tb.nextValidTime.Sub(tb.lastRecordedTime)
tb.deltatime = time.Duration(diff / time.Duration(tb.noTimeCount+1))
@ -74,7 +73,7 @@ func (tb *InferTimeBlock) finalizeTimeGap() error {
}
//validateTimeGap checks some important assumptions
func (tb *InferTimeBlock) validateTimeGap() error{
func (tb *InferTimeBlock) validateTimeGap() error {
//Check that lastRecordedTime and nextValidTime are not null
if tb.lastRecordedTime.IsZero() {
return errors.New("Gap start time is empty")

@ -395,8 +395,6 @@ func TestLoadFile_redefining_myCall_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "myCall on4do")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -435,8 +433,6 @@ func TestLoadFile_redefining_myWWFF_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "myWWFF onff-0001")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -475,8 +471,6 @@ func TestLoadFile_redefining_mySOTA_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "mySota on/on-111")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -516,8 +510,6 @@ func TestLoadFile_redefining_myGRID_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "myGrid ZZ99")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -540,8 +532,6 @@ func TestLoadFile_redefining_myGRID_must_fail(t *testing.T) {
os.Remove(temporaryDataFileName)
}
func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
//Given
@ -558,8 +548,6 @@ func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "operator blahh")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -582,7 +570,6 @@ func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
os.Remove(temporaryDataFileName)
}
func TestLoadFile_redefining_nickname_must_fail(t *testing.T) {
//Given
@ -599,8 +586,6 @@ func TestLoadFile_redefining_nickname_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "nickname blaaahh")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -1021,8 +1006,6 @@ func TestLoadFile_2_QSO_same_time(t *testing.T) {
os.Remove(temporaryDataFileName)
}
func TestLoadFile_wrongData(t *testing.T) {
//Given

@ -154,7 +154,6 @@ func ParseLine(inputStr string, previousLine LogLine) (logLine LogLine, errorMsg
//Scan the + part
if regexpDayIncrementPattern.MatchString(element) {
increment := len(element)
fmt.Println(logLine.Date)
newDate, dateError := IncrementDate(logLine.Date, increment)
if dateError != "" {
errorMsg = errorMsg + fmt.Sprintf(dateError)

@ -178,6 +178,11 @@ func TestParseLine(t *testing.T) {
args{inputStr: "day ++ 1230 oe6cud/p ", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},
LogLine{Date: "2020-09-07", Call: "OE6CUD/P", Time: "1230", ActualTime: "1230", RSTsent: "59", RSTrcvd: "59", Mode: "FM", ModeType: "PHONE"}, "",
},
{
"date processing - day single line",
args{inputStr: "day ++", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},
LogLine{Date: "2020-09-07", RSTsent: "59", RSTrcvd: "59", Mode: "FM", ModeType: "PHONE"}, "",
},
{
"date processing - day (error) ",
args{inputStr: "day +++++++++++ 1230 oe6cud/p ", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},

@ -4,29 +4,22 @@
@test "Does the version show?" {
output=$(docker run --rm on4kjm/flecli:latest version -d)
echo 'status:' $status
echo 'output:' $output
# [ "$status" -eq 0 ]
}
@test "Can a simple file be loaded?" {
output=$(test/docker-FLEcli.sh load -i test/data/fle-1.txt)
echo 'status:' $status
echo 'output:' $output
}
@test "Can a more complex file be loaded?" {
output=$(test/docker-FLEcli.sh load -i test/data/ON4KJM@ONFF-025920200524.txt)
echo 'status:' $status
echo 'output:' $output
}
@test "Is the generated SOTA csv equivalent to the canonical one?" {
mkdir -p test/output/temp
output=$(test/docker-FLEcli.sh csv -o -i test/FLE-sample/sota_wwff.txt test/output/temp/sota_wwff.csv)
echo 'status:' $status
echo 'output:' $output
diff test/output/temp/sota_wwff.csv test/FLE-sample/sota_wwff.csv --strip-trailing-cr
}
@ -39,3 +32,7 @@
run test/docker-FLEcli.sh csv -o -i test/data/fle-5-wrong-call.txt
[ "$status" -eq 1 ]
}
@test "Processing a big FLE file" {
run test/docker-FLEcli.sh csv -o -i test/data/fle-6-bigFile.txt test/output/temp/fle-6-bigFile.csv
}

File diff suppressed because it is too large Load Diff

@ -1,18 +0,0 @@
#!/bin/bash
set -e
./build.sh
echo "--------------------------"
./FLEcli load -i test/data/fle-1.txt
echo "--------------------------"
./FLEcli load -i test/data/ON4KJM@ONFF-025920200524.txt
echo "--------------------------"
mkdir -p test/output/temp
./FLEcli csv -o -i test/FLE-sample/sota_wwff.txt test/output/temp/sota_wwff.csv
diff test/output/temp/sota_wwff.csv test/FLE-sample/sota_wwff.csv --strip-trailing-cr && echo "no difference" || echo "differences!"
echo "--------------------------"
./FLEcli -i test/FLE-sample/sota_wwff.txt adif -o=test/output/temp/sota_wwff.adif --interpolate --overwrite --wwff --sota
Loading…
Cancel
Save