source: ammosreader/sample_scripts/import_signals.py@ 276d0aa

"""I import .ppdw files from multiple subdirectories into a mongodb."""

import sys
import re
import argparse
import numpy as np
from datetime import datetime
from pathlib import Path
from pymongo import MongoClient
from ammosreader.PPDWReader import PPDWReader

parser = argparse.ArgumentParser()
parser.add_argument('-s', '--source', required=True, help='specify source for given signal')
parser.add_argument('-i', '--input-dir', required=True, help='specify source directory')

args = parser.parse_args()

source = args.source.upper()
input_dir = Path(args.input_dir)

if not re.match('[A-Z0-9]{2}', source):
    print("Source identifier", source, "invalid")
    sys.exit()

code = input_dir.name

# FIXME: Directory structure differs
# FIXME: get julian date, source identifier, signal number and ELINT-Code from input_dir path

# if not re.match('[A-Z][0-9]{3}[A-Z]', code):
#    print("ELINT-Code incorrect")
#    sys.exit(1)


if not input_dir.is_dir():
    print("Input dir invalid")
    sys.exit(1)

subdirs = [f for f in input_dir.iterdir() if f.is_dir()]

client = MongoClient('hackathon.kid.local', 27017)

for each_dir in subdirs:

    # print("Each dir", each_dir)
    dir_name = each_dir.name

    # if not re.match('[0-9]{5}', dir_name):
    #    print("Signal number incorrect")
    #    sys.exit(1)

    files_inside = [each for each in (each_dir.iterdir()) if (each.suffix in ['.idx', '.iqdw', '.ppdw'] and
                                                              each_dir.stem == dir_name)]

    # print("Files inside", files_inside)
    if len(files_inside) != 3:
        print(".idx, .iqdw or .ppdw file missing in", each_dir)
        sys.exit(1)

    iqdw_file = (input_dir / each_dir / each_dir.stem).with_suffix('.iqdw')
    ppdw_file = (input_dir / each_dir / each_dir.stem).with_suffix('.ppdw')
    # print("PPDW file", ppdw_file)
    idx_file = (input_dir / each_dir / each_dir.stem).with_suffix('.idx')

    ppdw_reader = PPDWReader(ppdw_file)
    ppdw_reader.read_all_frames_left()

    database = client['ELINT']
    ts = (ppdw_reader.container.start_time() - np.datetime64('1970-01-01T00:00:00')) / np.timedelta64(1, 's')
    time_tuple = datetime.utcfromtimestamp(ts).timetuple()
    julian_date_string = str(time_tuple.tm_year)[2:] + str(time_tuple.tm_yday).zfill(3)
    print("Julian", julian_date_string, str(ppdw_file))
    code_collection = database[input_dir.name]
    year_collection = code_collection[str(time_tuple.tm_year)]
    source_collection = year_collection[source]

    json_representation = ppdw_reader.container.to_json()
    json_representation['SIGNALNUMBER'] = dir_name
    json_representation['SOURCEFILE'] = str(ppdw_file)
    source_collection.insert_one(json_representation)