dominic/FLEcli
1
0
Fork 0
mirror of https://github.com/on4kjm/FLEcli.git synced 2025-01-18 13:01:09 +01:00
Code Issues Releases Activity

Add test to process large FLE file

Browse source
This commit is contained in:
Jean-Marc MEESSEN 2020-10-06 13:53:57 +02:00 committed by GitHub
parent 8d03018796
commit ce146e3407
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 6327 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
flecmd/csv.go
View file

@ -27,34 +27,39 @@ import (
var outputCsvFilename string
var isOverwriteCsv bool
var processCsvCommand = fleprocess.ProcessCsvCommand
var csvCmd = csvCmdConstructor()
// csvCmd is executed when choosing the csv option (load FLE file and generate csv file)
var csvCmd = &cobra.Command{
Use: "csv [flags] inputFile [outputFile]",
Short: "Generates a SOTA .csv file based on a FLE type shorthand logfile.",
// Long: `A longer description that spans multiple lines and likely contains examples
// and usage of using your command. For example:
func csvCmdConstructor() *cobra.Command {
return &cobra.Command{
Use: "csv [flags] inputFile [outputFile]",
Short: "Generates a SOTA .csv file based on a FLE type shorthand logfile.",
// Long: `A longer description that spans multiple lines and likely contains examples
// and usage of using your command. For example:
RunE: func(cmd *cobra.Command, args []string) error {
//if args is empty, throw an error (Cobra will display the )
if len(args) == 0 {
//TODO: fix this ugly statement (because I am lazy)
return fmt.Errorf("Missing input file %s", "")
}
inputFilename = args[0]
if len(args) == 2 {
outputCsvFilename = args[1]
}
if len(args) > 2 {
return fmt.Errorf("Too many arguments.%s", "")
}
RunE: func(cmd *cobra.Command, args []string) error {
//if args is empty, throw an error (Cobra will display the )
if len(args) == 0 {
//TODO: fix this ugly statement (because I am lazy)
return fmt.Errorf("Missing input file %s", "")
}
inputFilename = args[0]
if len(args) == 2 {
outputCsvFilename = args[1]
}
if len(args) > 2 {
return fmt.Errorf("Too many arguments.%s", "")
}
if err := fleprocess.ProcessCsvCommand(inputFilename, outputCsvFilename, isInterpolateTime, isOverwriteCsv); err != nil {
fmt.Println("\nUnable to generate CSV file:")
fmt.Println(err)
os.Exit(1)
}
return nil
},
if err := fleprocess.ProcessCsvCommand(inputFilename, outputCsvFilename, isInterpolateTime, isOverwriteCsv); err != nil {
fmt.Println("\nUnable to generate CSV file:")
fmt.Println(err)
os.Exit(1)
}
return nil
},
}
}
func init() {

46
flecmd/load.go
View file

@ -19,33 +19,33 @@ limitations under the License.
import (
"FLEcli/fleprocess"
"fmt"
"github.com/spf13/cobra"
)
// loadCmd represents the load command
var loadCmd = &cobra.Command{
Use: "load [flags] inputFile",
Short: "Loads and validates a FLE type shorthand logfile",
// Long: `A longer description that spans multiple lines and likely contains examples
// and usage of using your command. For example:
var processLoadFile = fleprocess.LoadFile
var loadCmd = loadCmdConstructor()
// Cobra is a CLI library for Go that empowers applications.
// This application is a tool to generate the needed files
// to quickly create a Cobra application.`,
RunE: func(cmd *cobra.Command, args []string) error {
//if args is empty, throw an error
if len(args) == 0 {
//TODO: fix this ugly statement (because I am lazy)
return fmt.Errorf("Missing input file %s", "")
}
if len(args) > 1 {
return fmt.Errorf("Too many arguments.%s", "")
}
inputFilename = args[0]
fleprocess.LoadFile(inputFilename, isInterpolateTime)
return nil
},
// loadCmd represents the load command
func loadCmdConstructor() *cobra.Command {
return &cobra.Command{
Use: "load [flags] inputFile",
Short: "Loads and validates a FLE type shorthand logfile",
RunE: func(cmd *cobra.Command, args []string) error {
//if args is empty, throw an error
if len(args) < 1 {
//FIXME: Doesn't work as expected
return fmt.Errorf("Missing input file %s", "")
}
if len(args) > 1 {
return fmt.Errorf("Too many arguments.%s", "")
}
inputFilename = args[0]
//FIXME: we should return the result of the call
processLoadFile(inputFilename, isInterpolateTime)
return nil
},
}
}
func init() {

111
flecmd/load_test.go Normal file
View file

@ -0,0 +1,111 @@
package flecmd
/*
Copyright © 2020 Jean-Marc Meessen, ON4KJM <on4kjm@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import (
"FLEcli/fleprocess"
"bytes"
"fmt"
"os"
"strings"
"io/ioutil"
"testing"
)
func Test_ExecuteCommand_help(t *testing.T) {
cmd := loadCmdConstructor()
b := bytes.NewBufferString("")
cmd.SetOut(b)
cmd.SetArgs([]string{"--help"})
cmd.Execute()
out, err := ioutil.ReadAll(b)
if err != nil {
t.Fatal(err)
}
expectedOutput := "Loads and validates a FLE type shorthand logfile\n\nUsage:\n load [flags] inputFile\n\nFlags:\n -h, --help help for load\n"
if string(out) != expectedOutput {
t.Fatalf("expected \"%s\" got \"%s\"", expectedOutput, string(out))
}
}
func Test_ExecuteCommand_noArgs(t *testing.T) {
cmd := loadCmdConstructor()
b := bytes.NewBufferString("")
cmd.SetOut(b)
//cmd.SetArgs([]string{""})
cmd.Execute()
out, err := ioutil.ReadAll(b)
if err != nil {
t.Fatal(err)
}
//FIXME: doesn't work as espected
expectedOutputStart := "Error: Missing input file \nUsage:"
if !strings.HasPrefix(string(out), expectedOutputStart) {
t.Fatalf("expected to start with \"%s\" got \"%s\"", expectedOutputStart, string(out))
}
}
func Test_ExecuteCommand_toManyArgs(t *testing.T) {
cmd := loadCmdConstructor()
b := bytes.NewBufferString("")
cmd.SetOut(b)
cmd.SetArgs([]string{"blaah", "blaah", "blaah"})
cmd.Execute()
out, err := ioutil.ReadAll(b)
if err != nil {
t.Fatal(err)
}
expectedOutputStart := "Error: Too many arguments.\nUsage"
if !strings.HasPrefix(string(out), expectedOutputStart) {
t.Fatalf("expected to start with \"%s\" got \"%s\"", expectedOutputStart, string(out))
}
}
func Test_ExecuteCommand_happyCase(t *testing.T) {
processLoadFile = mockLoadFile
//Capture output
rescueStdout := os.Stdout
r, w, _ := os.Pipe()
os.Stdout = w
cmd := loadCmdConstructor()
cmd.SetArgs([]string{"data.txt"})
cmdErr := cmd.Execute()
//Close the capture and get the data
w.Close()
out, _ := ioutil.ReadAll(r)
os.Stdout = rescueStdout
if cmdErr != nil {
t.Fatalf("Unexpected error executing command: %s", cmdErr)
}
if string(out) != "fileLoad via mock" {
t.Fatalf("Expected \"fileLoad via mock\". Got \"%s\"", string(out))
}
}
func mockLoadFile(inputFilename string, isInterpolateTime bool) (filleFullLog []fleprocess.LogLine, isProcessedOK bool) {
fmt.Print("fileLoad via mock")
return nil, true
}

16
fleprocess/adif_process_test.go
View file

@ -1,5 +1,21 @@
package fleprocess
/*
Copyright © 2020 Jean-Marc Meessen, ON4KJM <on4kjm@gmail.com>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import (
"fmt"
"testing"

2
fleprocess/adif_write.go
View file

@ -70,7 +70,7 @@ func buildAdif(fullLog []LogLine, isWWFF bool, isSOTA bool) (adifList []string)
adifLine.WriteString(adifElement("MY_SIG_INFO", logLine.MyWWFF))
if logLine.WWFF != "" {
adifLine.WriteString(adifElement("SIG", "WWFF"))
adifLine.WriteString(adifElement("SIG_INFO",logLine.WWFF))
adifLine.WriteString(adifElement("SIG_INFO", logLine.WWFF))
}
}
if isSOTA {

11
fleprocess/csv_process.go
View file

@ -111,20 +111,19 @@ func validateDataForSotaCsv(loadedLogFile []LogLine) error {
}
errorsBuffer.WriteString(fmt.Sprintf("missing QSO time %s", errorLocation))
}
//FIXME: if isNoMySota and MySota defined means that it was defined later in the log file
if (isNoMySota && loadedLogFile[i].MySOTA != "") {
//FIXME: if isNoMySota and MySota defined means that it was defined later in the log file
if isNoMySota && loadedLogFile[i].MySOTA != "" {
if errorsBuffer.String() != "" {
errorsBuffer.WriteString(fmt.Sprintf(", "))
}
errorsBuffer.WriteString(fmt.Sprintf("encountered an unexpexted MySota reference while processing what should be a chaser log %s", errorLocation))
errorsBuffer.WriteString(fmt.Sprintf("encountered an unexpexted MySota reference while processing what should be a chaser log %s", errorLocation))
}
if (isNoMySota && loadedLogFile[i].SOTA == "") {
if isNoMySota && loadedLogFile[i].SOTA == "" {
if errorsBuffer.String() != "" {
errorsBuffer.WriteString(fmt.Sprintf(", "))
}
errorsBuffer.WriteString(fmt.Sprintf("missing SOTA reference while attempting to process chaser log %s", errorLocation))
errorsBuffer.WriteString(fmt.Sprintf("missing SOTA reference while attempting to process chaser log %s", errorLocation))
}
}
if errorsBuffer.String() != "" {

5
fleprocess/inferTime.go
View file

@ -54,11 +54,10 @@ func (tb *InferTimeBlock) String() string {
//finalizeTimeGap makes the necessary checks and computation
func (tb *InferTimeBlock) finalizeTimeGap() error {
if err :=tb.validateTimeGap(); err != nil {
if err := tb.validateTimeGap(); err != nil {
return err
}
//Compute the gap
diff := tb.nextValidTime.Sub(tb.lastRecordedTime)
tb.deltatime = time.Duration(diff / time.Duration(tb.noTimeCount+1))
@ -74,7 +73,7 @@ func (tb *InferTimeBlock) finalizeTimeGap() error {
}
//validateTimeGap checks some important assumptions
func (tb *InferTimeBlock) validateTimeGap() error{
func (tb *InferTimeBlock) validateTimeGap() error {
//Check that lastRecordedTime and nextValidTime are not null
if tb.lastRecordedTime.IsZero() {
return errors.New("Gap start time is empty")

23
fleprocess/load_file_test.go
View file

@ -395,8 +395,6 @@ func TestLoadFile_redefining_myCall_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "myCall on4do")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -435,8 +433,6 @@ func TestLoadFile_redefining_myWWFF_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "myWWFF onff-0001")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -475,8 +471,6 @@ func TestLoadFile_redefining_mySOTA_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "mySota on/on-111")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -516,8 +510,6 @@ func TestLoadFile_redefining_myGRID_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "myGrid ZZ99")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -540,8 +532,6 @@ func TestLoadFile_redefining_myGRID_must_fail(t *testing.T) {
os.Remove(temporaryDataFileName)
}
func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
//Given
@ -558,8 +548,6 @@ func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "operator blahh")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -582,7 +570,6 @@ func TestLoadFile_redefining_operator_must_fail(t *testing.T) {
os.Remove(temporaryDataFileName)
}
func TestLoadFile_redefining_nickname_must_fail(t *testing.T) {
//Given
@ -599,8 +586,6 @@ func TestLoadFile_redefining_nickname_must_fail(t *testing.T) {
dataArray = append(dataArray, "40m cw 0950 ik5zve/5 9 5")
dataArray = append(dataArray, "nickname blaaahh")
temporaryDataFileName := createTestFile(dataArray)
//When
@ -823,7 +808,7 @@ func TestLoadFile_InferTime_missingStartTime(t *testing.T) {
if loadedLogFile[0].MyWWFF != expectedValue {
t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue)
}
expectedValue = "IK5ZVE"
if loadedLogFile[0].Call != expectedValue {
t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
@ -896,7 +881,7 @@ func TestLoadFile_InferTime_missingEndTime(t *testing.T) {
if loadedLogFile[0].MyWWFF != expectedValue {
t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue)
}
expectedValue = "IK5ZVE"
if loadedLogFile[0].Call != expectedValue {
t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
@ -972,7 +957,7 @@ func TestLoadFile_2_QSO_same_time(t *testing.T) {
if loadedLogFile[0].MyWWFF != expectedValue {
t.Errorf("Not the expected MyWWFF value: %s (expecting %s)", loadedLogFile[0].MyWWFF, expectedValue)
}
expectedValue = "IK5ZVE"
if loadedLogFile[0].Call != expectedValue {
t.Errorf("Not the expected Call[0] value: %s (expecting %s)", loadedLogFile[0].Call, expectedValue)
@ -1021,8 +1006,6 @@ func TestLoadFile_2_QSO_same_time(t *testing.T) {
os.Remove(temporaryDataFileName)
}
func TestLoadFile_wrongData(t *testing.T) {
//Given

1
fleprocess/parse_line.go
View file

@ -154,7 +154,6 @@ func ParseLine(inputStr string, previousLine LogLine) (logLine LogLine, errorMsg
//Scan the + part
if regexpDayIncrementPattern.MatchString(element) {
increment := len(element)
fmt.Println(logLine.Date)
newDate, dateError := IncrementDate(logLine.Date, increment)
if dateError != "" {
errorMsg = errorMsg + fmt.Sprintf(dateError)

7
fleprocess/parse_line_test.go
View file

@ -57,7 +57,7 @@ func TestParseLine(t *testing.T) {
"Parse suspecious line",
args{inputStr: "1029 Sm/dl8mf 559 579", previousLine: LogLine{Time: "1200", Mode: "CW", ModeType: "CW"}},
LogLine{Time: "1029", ActualTime: "1029", Call: "SM/DL8MF", Mode: "CW", ModeType: "CW", RSTsent: "559", RSTrcvd: "579"}, "",
},
},
{
"Parse partial time - 3",
args{inputStr: "4 g3noh", previousLine: LogLine{Time: "1200", Mode: "SSB"}},
@ -178,6 +178,11 @@ func TestParseLine(t *testing.T) {
args{inputStr: "day ++ 1230 oe6cud/p ", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},
LogLine{Date: "2020-09-07", Call: "OE6CUD/P", Time: "1230", ActualTime: "1230", RSTsent: "59", RSTrcvd: "59", Mode: "FM", ModeType: "PHONE"}, "",
},
{
"date processing - day single line",
args{inputStr: "day ++", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},
LogLine{Date: "2020-09-07", RSTsent: "59", RSTrcvd: "59", Mode: "FM", ModeType: "PHONE"}, "",
},
{
"date processing - day (error) ",
args{inputStr: "day +++++++++++ 1230 oe6cud/p ", previousLine: LogLine{Date: "2020-09-05", Mode: "FM", ModeType: "PHONE"}},

2
fleprocess/validate_test.go
View file

@ -191,7 +191,7 @@ func TestValidateCall(t *testing.T) {
"Valid call from activation (case 2)",
args{sign: "Sm/dl8mf"},
"SM/DL8MF", "",
},
},
{
"Valid prefix (issue #2)",
args{sign: "e7/z35m/p"},

13
test/bats-scripts/test.bats
View file

@ -4,29 +4,22 @@
@test "Does the version show?" {
output=$(docker run --rm on4kjm/flecli:latest version -d)
echo 'status:' $status
echo 'output:' $output
# [ "$status" -eq 0 ]
}
@test "Can a simple file be loaded?" {
output=$(test/docker-FLEcli.sh load -i test/data/fle-1.txt)
echo 'status:' $status
echo 'output:' $output
}
@test "Can a more complex file be loaded?" {
output=$(test/docker-FLEcli.sh load -i test/data/ON4KJM@ONFF-025920200524.txt)
echo 'status:' $status
echo 'output:' $output
}
@test "Is the generated SOTA csv equivalent to the canonical one?" {
mkdir -p test/output/temp
output=$(test/docker-FLEcli.sh csv -o -i test/FLE-sample/sota_wwff.txt test/output/temp/sota_wwff.csv)
echo 'status:' $status
echo 'output:' $output
diff test/output/temp/sota_wwff.csv test/FLE-sample/sota_wwff.csv --strip-trailing-cr
}
@ -38,4 +31,8 @@
@test "Processing a FLE file with parsing errors must fail!" {
run test/docker-FLEcli.sh csv -o -i test/data/fle-5-wrong-call.txt
[ "$status" -eq 1 ]
}
@test "Processing a big FLE file" {
run test/docker-FLEcli.sh csv -o -i test/data/fle-6-bigFile.txt test/output/temp/fle-6-bigFile.csv
}

6124
test/data/fle-6-bigFile.txt Normal file
View file

File diff suppressed because it is too large Load diff

18
test/test.sh
View file

@ -1,18 +0,0 @@
#!/bin/bash
set -e
./build.sh
echo "--------------------------"
./FLEcli load -i test/data/fle-1.txt
echo "--------------------------"
./FLEcli load -i test/data/ON4KJM@ONFF-025920200524.txt
echo "--------------------------"
mkdir -p test/output/temp
./FLEcli csv -o -i test/FLE-sample/sota_wwff.txt test/output/temp/sota_wwff.csv
diff test/output/temp/sota_wwff.csv test/FLE-sample/sota_wwff.csv --strip-trailing-cr && echo "no difference" || echo "differences!"
echo "--------------------------"
./FLEcli -i test/FLE-sample/sota_wwff.txt adif -o=test/output/temp/sota_wwff.adif --interpolate --overwrite --wwff --sota