ssscoring.flysight

  1# See: https://github.com/pr3d4t0r/SSScoring/blob/master/LICENSE.txt
  2
  3"""
  4Functions and logic for detecting, validating and manipulating
  5FlySight CSV files, including detection in the file system.  The functions in
  6this module assume that a data lake exists somewhere in the file system (whether
  7local or cloud-based).
  8"""
  9
 10
 11from collections import OrderedDict
 12from io import StringIO
 13from pathlib import Path
 14
 15from ssscoring.constants import FLYSIGHT_1_HEADER
 16from ssscoring.constants import FLYSIGHT_2_HEADER
 17from ssscoring.constants import FLYSIGHT_FILE_ENCODING
 18from ssscoring.constants import INSIGHT_1_HEADER
 19from ssscoring.constants import IGNORE_LIST
 20from ssscoring.constants import MIN_JUMP_FILE_SIZE
 21from ssscoring.datatypes import FlySightVersion
 22from ssscoring.errors import SSScoringError
 23
 24import csv
 25import os
 26import shutil
 27import tempfile
 28
 29import pandas as pd
 30
 31
 32# +++ functions +++
 33
 34def _isInsightHeader(header: list) -> bool:
 35    return INSIGHT_1_HEADER.issubset(set(header)) and 'cAcc' not in header
 36
 37
 38def isCRMangledCSV(fileThing) -> bool:
 39    """
 40    Tests if `fileThing` is an Excel or Dropbox DOS file with lines terminated
 41    in CRCRLF.  These occur when someone opens the file with Excel or some other
 42    tool in a Windows system and saves the file back to the file system,
 43    mangling the original format.
 44
 45    Arguments
 46    ---------
 47        fileThing
 48    A string or `pathlib.Path` object associated with what looks like a FlySight
 49    CR mangled file.
 50
 51    Returns
 52    -------
 53    `True` if the file has one or more lines ending in CRCRLF within the first
 54    512 bytes of data.
 55    """
 56    with open (fileThing, 'rb') as file:
 57        rawData = file.read()
 58        return b'\r\r\n' in rawData
 59
 60
 61def fixCRMangledCSV(fileThing):
 62    """
 63    Open the file associated with `fileThing` and repleace all`\r\r\b` with
 64    `\r\n` EOL markers.
 65
 66    Arguments
 67    ---------
 68        fileThing
 69    A string or `pathlib.Path` object associated with what looks like a FlySight
 70    CR mangled file.
 71
 72    See
 73    ---
 74    `ssscoring.flysight.isCRMangledCSV`
 75    """
 76    with open(fileThing, 'rb') as inputFile:
 77        fileContents = inputFile.read()
 78    fileContents = fileContents.replace(b'\r\r\n', b'\r\n')
 79    with tempfile.NamedTemporaryFile(delete = False) as outputFile:
 80        outputFile.write(fileContents)
 81        tempFileName = outputFile.name
 82    shutil.copy(tempFileName, fileThing)
 83    os.unlink(tempFileName)
 84
 85
 86def skipOverFS2MetadataRowsIn(data: pd.DataFrame) -> pd.DataFrame:
 87    """
 88    Returns a clean dataframe on which any metadata rows within the first 100
 89    are skipped.  This function uses the `time` column to detect valid rows.  A
 90    `time == NaN` is considered invalid and skipped.
 91
 92    Arguments
 93    ---------
 94        data
 95    A FlySight 2 dataframe suspected of having dirty N first rows with metadata
 96
 97    Returns
 98    -------
 99    A FlySight 2 clean dataframe without any leading metadata rows.
100    """
101    for ref in range(0,100):
102        if pd.notnull(data.iloc[ref].time):
103            break
104    return data.iloc[ref:]
105
106
107def validFlySightHeaderIn(fileThingCSV) -> bool:
108    """
109    Checks if a file is a CSV in FlySight 1 or FlySight 2 formats.  The checks
110    include:
111
112    - Whether the file is a CSV, using a comma delimiter
113    - Checks for the presence of all the documented FlySight 1 headers
114    - Checks for the presence of the FlySight 2 line 1 identifier
115
116    Arguments
117    ---------
118        fileThingCSV
119    A file thing to verify as a valid FlySight file; can be a string, an
120    instance of `libpath.Path`, or a buffer of `bytes`.
121
122    Returns
123    -------
124    `True` if `fileThingCSV` is a FlySight CSV file, otherwise `False`.
125    """
126    delimiters = [',']
127
128    if isinstance(fileThingCSV, bytes):
129        stream = StringIO(fileThingCSV.decode(FLYSIGHT_FILE_ENCODING))
130    else:
131        stream = open(fileThingCSV, 'r')
132
133    with stream:
134        try:
135            dialect = csv.Sniffer().sniff(stream.readline(), delimiters=delimiters)
136        except csv.Error:
137            return False
138
139        if dialect.delimiter not in delimiters:
140            return False
141        stream.seek(0)
142        try:
143            header = next(csv.reader(stream))
144        except StopIteration:
145            return False
146    return header[0] == '$FLYS' or FLYSIGHT_1_HEADER.issubset(header) or _isInsightHeader(header)
147
148
149def getAllSpeedJumpFilesFrom(dataLake: Path) -> dict:
150    """
151    Get a list of all the speed jump files from a data lake, where data lake is
152    defined as a reachable path that contains one or more FlySight CSV files.
153    This function tests each file to ensure that it's a speed skydive FlySight
154    file in a valid format and length.  It doesn't validate data like versions
155    prior to 1.9.0.
156
157    Arguments
158    ---------
159        dataLake: str
160    A valid (absolute or relative) path name to the top level directory where
161    the data lake starts.
162
163    Returns
164    -------
165    A dictionary of speed jump file names for later SSScoring processing:
166        - keys are the file names
167        - values are a FlySight version string tag
168    """
169    jumpFiles = OrderedDict()
170    for root, dirs, files in os.walk(dataLake):
171        if any(name in root for name in IGNORE_LIST):
172            continue
173        for fileName in files:
174            data = None
175            if '.swp' in fileName: # Ignore Vim, other editors swap file
176                continue
177            if '.CSV' in fileName.upper():
178                version = '1'
179                jumpFileName = Path(root) / fileName
180                stat = os.stat(jumpFileName)
181                if all(x not in fileName for x in ('EVENT', 'SENSOR', 'TRACK')):
182                    # FlySight 1 or Insight track format
183                    data = pd.read_csv(jumpFileName, skiprows = (1, 1), index_col = False)
184                    if data is not None and 'headAcc' in data.columns:
185                        version = 'i'
186                elif 'TRACK' in fileName:
187                    # FlySight 2 track custom format
188                    data = pd.read_csv(jumpFileName, names = FLYSIGHT_2_HEADER, skiprows = 6, index_col = False, na_values = ['NA', ])
189                    data = skipOverFS2MetadataRowsIn(data)
190                    data.drop('GNSS', inplace = True, axis = 1)
191                    version = '2'
192                if data is not None and stat.st_size >= MIN_JUMP_FILE_SIZE and validFlySightHeaderIn(jumpFileName):
193                    # explicit because `not data` is ambiguous for dataframes
194                    jumpFiles[jumpFileName] = version
195    jumpFiles = OrderedDict(sorted(jumpFiles.items()))
196    return jumpFiles
197
198
199def detectFlySightFileVersionOf(fileThing) -> FlySightVersion:
200    """
201    Detects the FlySight file version based on its file name and format.
202
203    Arguments
204    ---------
205        fileThing
206    A string, `bytes` buffer or `pathlib.Path` object corresponding to track
207    file.  If string or `pathlib.Path`, it'll be treated as a file.
208
209    Returns
210    -------
211    An instance of `ssscoring.flysight.FlySightVersion` with a valid version
212    symbolic value.
213
214    Errors
215    ------
216    `ssscoring.errors.SSScoringError` if the file is not a CSV and it's some
217    other invalid format.
218    """
219    match fileThing:
220        case Path():
221            fileName = fileThing.as_posix()
222        case str():
223            fileName = fileThing
224            fileThing = Path(fileThing)
225        case bytes():
226            fileName = '00-00-00.CSV'
227        case _:
228            raise SSScoringError('fileThing must be a Path, str, or bytes')
229
230    delimiters =  [',', ]
231    stream = None
232    if not '.CSV' in fileName.upper():
233        raise SSScoringError('Invalid file extension type')
234    if any(x in fileName for x in ('EVENT.CSV', 'SENSOR.CSV')):
235        raise SSScoringError('Only TRACK.CSV v2 files can be processed at this time')
236    if isinstance(fileThing, Path) or isinstance(fileThing, str):
237        if not fileThing.is_file():
238            raise SSScoringError('%s - file not found in data lake' % fileName)
239        if not validFlySightHeaderIn(fileName):
240            raise SSScoringError('CSV is not a valid FlySight file')
241        stream = open(fileName, 'r')
242    elif isinstance(fileThing, bytes):
243        stream = StringIO(fileThing.decode(FLYSIGHT_FILE_ENCODING))
244
245    try:
246        dialect = csv.Sniffer().sniff(stream.readline(), delimiters = delimiters)
247    except:
248        raise SSScoringError('Error while trying to validate %s file format' % fileName)
249    if dialect.delimiter in delimiters:
250        stream.seek(0)
251        header = next(csv.reader(stream))
252    else:
253        raise SSScoringError('CSV uses a different delimiter from FlySigh')
254    if header[0] == '$FLYS':
255        return FlySightVersion.V2
256    elif FLYSIGHT_1_HEADER.issubset(header):
257        return FlySightVersion.V1
258    elif _isInsightHeader(header):
259        return FlySightVersion.INSIGHT
260    else:
261        raise SSScoringError('%s file is not a FlySight v1 or v2 file')
262
263
264def readVersion1CSV(fileThing: object) -> pd.DataFrame:
265    """
266    Read a FlySight file version 1 into a dataframe.  It scrubes blank rows that
267    get in the way of correct parsing.
268
269    Arguments
270    ---------
271        fileThing
272    A string or a `pathlib.Path` object.  It can be a relative or an absolute
273    path.
274
275    Returns
276    -------
277    A FlySight dataframe with the original column names, normalized for
278    manipulation as a dataframe instead of a file or CSV object.
279    """
280    return pd.read_csv(fileThing, skiprows = (1, 1), index_col = False)
281
282
283def _tagVersion1From(fileThing: str) -> str:
284    return fileThing.replace('.CSV', '').replace('.csv', '').replace('/data', '').replace('/', ' ').strip()+':v1'
285
286
287def _tagFromFirstTimestampIn(rawData: pd.DataFrame, suffix: str) -> str:
288    firstTimestamp = str(rawData.iloc[0]['time'])
289    return firstTimestamp.split('T')[1].split('.')[0].replace(':', '-')+':'+suffix
290
291
292def _tagVersion2From(rawData: pd.DataFrame) -> str:
293    return _tagFromFirstTimestampIn(rawData, 'v2')
294
295
296def _tagInsightFrom(rawData: pd.DataFrame) -> str:
297    return _tagFromFirstTimestampIn(rawData, 'i')
298
299
300def readInsightCSV(fileThing: object) -> pd.DataFrame:
301    rawData = pd.read_csv(fileThing, skiprows=(1, 1), index_col=False)
302    rawData.drop('headAcc', inplace=True, axis=1)
303    return rawData
304
305
306def readVersion2CSV(jumpFile: str) -> pd.DataFrame:
307    """
308    Read a FlySight file version 2 into a dataframe.  It scrubes blank rows that
309    get in the way of correct parsing and drops the `GNSS` column because it
310    just makes dataframe management murkier.
311
312    Arguments
313    ---------
314        fileThing
315    A string or a `pathlib.Path` object.  It can be a relative or an absolute
316    path.
317
318    Returns
319    -------
320    A FlySight dataframe with the original column names, normalized for
321    manipulation as a dataframe instead of a file or CSV object.
322    """
323
324    rawData = pd.read_csv(jumpFile, names = FLYSIGHT_2_HEADER, skiprows = 6, index_col = False, na_values=['NA',])
325    rawData = skipOverFS2MetadataRowsIn(rawData)
326    rawData.drop('GNSS', inplace = True, axis = 1)
327    return rawData
328
329
330def getFlySightDataFromCSVBuffer(buffer:bytes, bufferName:str) -> tuple:
331    """
332    Ingress a buffer with known FlySight or SkyTrax file data for SSScoring
333    processing.
334
335    Arguments
336    ---------
337        buffer
338    A binary data buffer, bag of bytes, containing a known FlySight track file.
339
340        bufferName
341    An arbitrary name for the buffer of type `str`.  Used to construct the tag
342    for FlySight 1 buffers; ignored for FlySight 2 and Insight buffers, whose
343    tags are derived from the first row's timestamp.
344
345    Returns
346    -------
347    A `tuple` with two items:
348        - `rawData` - a dataframe representation of the CSV with the original
349          headers but without the data type header
350        - `tag` - an identifying string for the track.  Shape depends on the
351          device:
352            - FlySight 1: `<bufferName>:v1` - derived from `bufferName`
353            - FlySight 2: `HH-MM-ss:v2` - derived from the first GNSS row's
354              timestamp
355            - Insight: `HH-MM-ss:i` - derived from the first row's timestamp
356          Invalid files produce `<bufferName>:INVALID`.
357
358    Raises
359    ------
360    `SSScoringError` if the CSV file is invalid in any way.
361    """
362    if not isinstance(buffer, bytes):
363        raise SSScoringError('buffer must be an instance of bytes, a bytes buffer')
364    try:
365        stringIO = StringIO(buffer.decode(FLYSIGHT_FILE_ENCODING))
366    except Exception as e:
367        raise SSScoringError('invalid buffer endcoding - %s' % str(e))
368    try:
369        version = detectFlySightFileVersionOf(buffer)
370    except Exception:
371        tag = '%s:INVALID' % bufferName
372        rawData = None
373    else:
374        if version == FlySightVersion.V1:
375            rawData = readVersion1CSV(stringIO)
376            tag = _tagVersion1From(bufferName)
377        elif version == FlySightVersion.V2:
378            rawData = readVersion2CSV(stringIO)
379            tag = _tagVersion2From(rawData)
380        elif version == FlySightVersion.INSIGHT:
381            rawData = readInsightCSV(stringIO)
382            tag = _tagInsightFrom(rawData)
383    return (rawData, tag)
384
385
386def getFlySightDataFromCSVFileName(jumpFile) -> tuple:
387    """
388    Ingress a known FlySight or SkyTrax file into memory for SSScoring
389    processing.
390
391    Arguments
392    ---------
393        jumpFile
394    A string or `pathlib.Path` object; can be a relative or an asbolute path.
395
396    Returns
397    -------
398    A `tuple` with two items:
399        - `rawData` - a dataframe representation of the CSV with the original
400          headers but without the data type header
401        - `tag` - an identifying string for the track.  Shape depends on the
402          device:
403            - FlySight 1: `<path slug>:v1` - derived from the file path
404            - FlySight 2: `HH-MM-ss:v2` - derived from the first GNSS row's
405              timestamp
406            - Insight: `HH-MM-ss:i` - derived from the first row's timestamp
407          `'NA'` if version detection fails.
408
409    Raises
410    ------
411    `SSScoringError` if the CSV file is invalid in any way.
412    """
413    if isinstance(jumpFile, Path):
414        jumpFile = jumpFile.as_posix()
415    elif isinstance(jumpFile, str):
416        pass
417    else:
418        raise SSScoringError('jumpFile must be a string or a Path object')
419    if not validFlySightHeaderIn(jumpFile):
420        raise SSScoringError('%s is an invalid speed skydiving file')
421    try:
422        version = detectFlySightFileVersionOf(jumpFile)
423    except Exception:
424        tag = 'NA'
425        rawData = None
426    else:
427        if version == FlySightVersion.V1:
428            rawData = readVersion1CSV(jumpFile)
429            tag = _tagVersion1From(jumpFile)
430        elif version == FlySightVersion.V2:
431            rawData = readVersion2CSV(jumpFile)
432            tag = _tagVersion2From(rawData)
433        elif version == FlySightVersion.INSIGHT:
434            rawData = readInsightCSV(jumpFile)
435            tag = _tagInsightFrom(rawData)
436    return (rawData, tag)