Index: pyproject.toml =================================================================== --- pyproject.toml (revision f67e593fcbfa076e39055aa0b6e7b01a469f040a) +++ pyproject.toml (revision bf33fd70c5a4abd1bbff19ca6989ec800a571e29) @@ -12,4 +12,5 @@ authors = [ { name = "Enrico Schwass", email="ennoausberlin@mac.com"} + ] description = "A package to manage ammos datastreams" @@ -17,5 +18,4 @@ requires-python = ">=3.9" -dependencies = ["numpy", "Pillow", "pydub", "bitstring", -"pymongo", "scipy"] +dependencies = ["numpy"] Index: quirements.txt =================================================================== --- requirements.txt (revision f67e593fcbfa076e39055aa0b6e7b01a469f040a) +++ (revision ) @@ -1,7 +1,0 @@ -bitstring==3.1.9 -numpy==1.23.0 -Pillow==9.2.0 -pydub==0.25.1 -pymongo==4.1.1 -scipy==1.8.1 -setuptools==58.1.0 Index: mple_scripts/ammos_viewer.py =================================================================== --- sample_scripts/ammos_viewer.py (revision f67e593fcbfa076e39055aa0b6e7b01a469f040a) +++ (revision ) @@ -1,109 +1,0 @@ -"""I provide a simple bit viewer specialized for AMMOS files.""" - -import tkinter as tk -from PIL import Image, ImageTk -# from random import randbytes -import bitstring - -from ammosreader.AmmosAudioReader import AmmosAudioReader - - -class MainApplication(tk.Frame): - """I implement a simple bit viewer specialized for AMMOS files.""" - - def __init__(self, parent): - """I create the application window.""" - super().__init__(parent) - - self.parent = parent - self.file_name = "../sample_data/audio_data_sample.bin" - self.ammos_container = AmmosAudioReader(self.file_name).read_all_frames_left() - self.file = open(self.file_name, 'rb') - self.bytes = self.file.read(-1) - self.cycle = tk.IntVar() - self.cycle.set(self.ammos_container.unique_frame_sizes()[0]) - self.cache = tk.IntVar() - self.cache.set(500) - self.canvas_size = (800, 600) - self.offset = tk.IntVar() - self.offset.set(0) - self.buildup() - - def buildup(self): - """I build up the initial user interface.""" - self.parent.title("File name:" + self.file_name + " Size:" + str(len(self.bytes))) - self.pack(fill="both", expand=True) - self.default_button_width = 10 - image_frame = tk.Frame(self) - image_frame.pack(side=tk.LEFT) - self.canvas = tk.Canvas(image_frame, bg="#000000", width=self.canvas_size[0], height=self.canvas_size[1]) - self.canvas.pack() - pil_image = Image.frombytes("1", self.canvas_size, self.current_bytes()) - self.image = ImageTk.PhotoImage(pil_image) - # pil_image.show() - button_frame = tk.Frame(self) - self.image_id = self.canvas.create_image(0, 0, anchor='nw', image=self.image) - button_frame.pack(side="right") - - tk.Label(button_frame, text='Cycle in bytes').pack(side=tk.TOP) - self.cycle_entry = tk.Entry(button_frame, text=self.cycle) - self.cycle_entry.pack(side=tk.TOP) - - tk.Label(button_frame, text='Cache in Megabytes').pack(side=tk.TOP) - self.cache_entry = tk.Entry(button_frame, text=self.cache) - self.cache_entry.pack(side=tk.TOP) - - tk.Label(button_frame, text='Offset in bytes').pack(side=tk.TOP) - self.offset_entry = tk.Entry(button_frame, text=self.offset) - self.offset_entry.pack(side=tk.TOP) - - tk.Button(button_frame, - text="Update", - width=self.default_button_width, - command=self.update_canvas).pack(side=tk.TOP, fill=tk.BOTH, expand=True) - - tk.Button( - button_frame, - text="Load file", - width=self.default_button_width, - command=self.load_file).pack(side="top", fill="both", expand=True) - - tk.Button( - button_frame, - text="Quit", - width=self.default_button_width, - command=self.quit).pack(side="top", fill="both", expand=True) - - def current_bytes(self): - """I return the current bytes to display in the canvas.""" - if self.offset.get() % 8 == 0: - return b"".join([self.bytes[each: each+self.canvas_size[0]] for each in range(self.offset.get(), - len(self.bytes), - self.cycle.get())]) - bits = bitstring.BitArray(self.bytes) - new_bytes = bits[self.offset.get()*8:].tobytes() - return b"".join([new_bytes[each: each+self.canvas_size[0]] for each in range(0, - len(new_bytes), - self.cycle.get())]) - - def update_canvas(self): - """I update the canvas.""" - self.image = ImageTk.PhotoImage(Image.frombytes("L", self.canvas_size, self.current_bytes())) - self.canvas.itemconfig(self.image_id, image=self.image) - - def load_file(self): - """I load a binary ammos file.""" - pass - - def quit(self): - """I quit the application.""" - self.parent.destroy() - - -if __name__ == '__main__': - root = tk.Tk() - root.geometry("1920x1200") - - app = MainApplication(root) - - root.mainloop() Index: mple_scripts/audio_reader.py =================================================================== --- sample_scripts/audio_reader.py (revision f67e593fcbfa076e39055aa0b6e7b01a469f040a) +++ (revision ) @@ -1,34 +1,0 @@ -import sys -import io -from pydub import AudioSegment -from pydub.playback import play - -from ammosreader.AmmosAudioReader import AmmosAudioReader - -if __name__ == '__main__': - - if len(sys.argv) != 2: - sys.exit() - - file_name = sys.argv[1] - - print("File name:", file_name) - - dat_file = AmmosAudioReader(file_name) - - dat_file.read_all_frames_left() - print("Sample rate:", dat_file.container.global_frames[0].global_frame_body.data_header.sample_rate) - print("Container size:", dat_file.container.size()) - print("Frequencies", dat_file.container.frequencies()) - print("Frame types:", dat_file.container.frame_types()) - print("Unique frame sizes", dat_file.container.unique_frame_sizes()) - print("Total frame size:", sum(dat_file.container.frame_sizes())) - print("Homogenic:", dat_file.container.is_homogenic()) - pcm_data = dat_file.pcm_for_channel(0) - print("PCM data size total:", len(pcm_data)) - data = AudioSegment.from_raw(io.BytesIO(pcm_data), - sample_width=2, - frame_rate=22050, - channels=1) - print("Start playing audio") - play(data) Index: mple_scripts/audio_socket_reader_test.py =================================================================== --- sample_scripts/audio_socket_reader_test.py (revision f67e593fcbfa076e39055aa0b6e7b01a469f040a) +++ (revision ) @@ -1,41 +1,0 @@ -import sys -import os - -sys.path.append('../src/') - -import socket - -import numpy as np -import scipy.io.wavfile as wavfile - -from ammosreader.AmmosAudioSocketReader import AmmosAudioSocketReader - - -if __name__ == '__main__': - frames = [] - - # create socket - in_socket = socket.socket() - in_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - in_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - in_socket.bind(('127.0.0.1', 12345)) - print('Waiting for connection') - in_socket.listen(1) - conn, _ = in_socket.accept() - print('Connected') - - # create AmmosAudioSocketReader - socket_reader = AmmosAudioSocketReader(conn) - - print('Reading Frames') - while True: - try: - frames.append(socket_reader.read_next_frame()) - except KeyboardInterrupt: - break - - # write frames to wav file - print('Writing frames to Wav File') - audio = np.concatenate([frame[0] for frame in frames]) - print(audio) - wavfile.write(f"test.wav", frames[0][1], audio) Index: mple_scripts/audio_socket_reader_test_streamer.py =================================================================== --- sample_scripts/audio_socket_reader_test_streamer.py (revision f67e593fcbfa076e39055aa0b6e7b01a469f040a) +++ (revision ) @@ -1,28 +1,0 @@ -import sys -import os -sys.path.append('../src/') -import socket - -try: - file_path = sys.argv[1] -except IndexError: - print("Please provide a file path as argument") - sys.exit(1) -except FileNotFoundError: - print("File not found") - sys.exit(1) - -file = open(file_path, 'rb') -total_bytes = file.read() -print('File read') - -print('Connecting to socket') -out_socket = socket.socket() -out_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -out_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) -out_socket.connect(('127.0.0.1', 12345)) -print('Connected') -print('Sending Bytes') -out_socket.sendall(total_bytes) -out_socket.close() -out_socket.shutdown(socket.SHUT_RDWR) Index: mple_scripts/import_signal.py =================================================================== --- sample_scripts/import_signal.py (revision f67e593fcbfa076e39055aa0b6e7b01a469f040a) +++ (revision ) @@ -1,94 +1,0 @@ -import sys -import re -import sqlite3 -import argparse -import numpy as np -from datetime import datetime -from pathlib import Path -from pymongo import MongoClient -from ammosreader.AmmosIFReader import AmmosIFReader -from ammosreader.PPDWReader import PPDWReader - -parser = argparse.ArgumentParser() -parser.add_argument('-s', '--source', required=True, help='specify source for given signal') -parser.add_argument('-i', '--input-dir', required=True, help='specify source directory') -parser.add_argument('-o', '--output-file', required=True, help='specify sqlite3 database file') -parser.add_argument('-n', '--signal-number', required=True, help='specify number for signal') -parser.add_argument('-c', '--code', required=True, help='specify code for signal') - -args = parser.parse_args() - -source = args.source.upper() -code = args.code.upper() -signal_number = args.signal_number -input_dir = Path(args.input_dir) -sqlite3_file = Path(args.output_file) - -if not re.match('[A-Z0-9]{2}', source): - print("Source identifier", source, "invalid") - sys.exit() - -if not re.match('[A-Z][0-9]{3}[A-Z]', code): - print("ELINT-Code incorrect") - sys.exit(1) - -if not re.match('[0-9]{4}', signal_number): - print("Signal number incorrect") - sys.exit(1) - -if not input_dir.is_dir(): - print("Input dir invalid") - sys.exit(1) - -dir_name = input_dir.name - -files_inside = [each for each in (input_dir.iterdir()) if (each.suffix in ['.idx', '.iqdw', '.ppdw'] and each.stem == dir_name)] - -if len(files_inside) != 3: - print(".idx, .iqdw or .ppdw file missing in", input_dir) - sys.exit(1) - -iqdw_file = (input_dir / input_dir.stem).with_suffix('.iqdw') -ppdw_file = (input_dir / input_dir.stem).with_suffix('.ppdw') -idx_file = (input_dir / input_dir.stem).with_suffix('.idx') - -#if not sqlite3_file.is_file(): -# print("Sqlite3 database file not found at", sqlite3_file) -# sys.exit(1) - -# try: -# connection = sqlite3.connect(str(sqlite3_file)) -# cursor = connection.cursor() -# except Exception: -# print("Can not connect to database file", str(sqlite3_file)) - # sys.exit(1) - -#ammos_if_reader = AmmosIFReader(str(iqdw_file)) -#ammos_if_reader.read_all_frames_left() -# for each_frame in ammos_if_reader.container.global_frames: -# print(each_frame) - -ppdw_reader = PPDWReader(ppdw_file) -ppdw_reader.read_all_frames_left() - -client = MongoClient('hackathon.kid.local', 27017) - -database = client['ELINT'] -ts = (ppdw_reader.container.start_time() - np.datetime64('1970-01-01T00:00:00')) / np.timedelta64(1, 's') -time_tuple = datetime.utcfromtimestamp(ts).timetuple() -julian_date_string = str(time_tuple.tm_year)[2:] + str(time_tuple.tm_yday).zfill(3) -code_collection = database[code] -year_collection = code_collection[str(time_tuple.tm_year)] -source_collection = year_collection[source] - -json_representation = ppdw_reader.container.to_json() -json_representation['SIGNALNUMBER'] = signal_number -json_representation['SOURCEFILE'] = str(ppdw_file) -source_collection.insert_one(json_representation) - -# for each in ppdw_reader.container.signals: -# each_json = each.to_json() -# each_json['SIGNALNUMBER'] = signal_number -# each_json['JULIANDATE'] = julian_date_string -# each_json['SOURCEFILE'] = str(ppdw_file) -# source_collection.insert_one(each_json) Index: mple_scripts/import_signals.py =================================================================== --- sample_scripts/import_signals.py (revision f67e593fcbfa076e39055aa0b6e7b01a469f040a) +++ (revision ) @@ -1,80 +1,0 @@ -"""I import .ppdw files from multiple subdirectories into a mongodb.""" - -import sys -import re -import argparse -import numpy as np -from datetime import datetime -from pathlib import Path -from pymongo import MongoClient -from ammosreader.PPDWReader import PPDWReader - -parser = argparse.ArgumentParser() -parser.add_argument('-s', '--source', required=True, help='specify source for given signal') -parser.add_argument('-i', '--input-dir', required=True, help='specify source directory') - -args = parser.parse_args() - -source = args.source.upper() -input_dir = Path(args.input_dir) - -if not re.match('[A-Z0-9]{2}', source): - print("Source identifier", source, "invalid") - sys.exit() - -code = input_dir.name - -# FIXME: Directory structure differs -# FIXME: get julian date, source identifier, signal number and ELINT-Code from input_dir path - -# if not re.match('[A-Z][0-9]{3}[A-Z]', code): -# print("ELINT-Code incorrect") -# sys.exit(1) - - -if not input_dir.is_dir(): - print("Input dir invalid") - sys.exit(1) - -subdirs = [f for f in input_dir.iterdir() if f.is_dir()] - -client = MongoClient('hackathon.kid.local', 27017) - -for each_dir in subdirs: - - # print("Each dir", each_dir) - dir_name = each_dir.name - - # if not re.match('[0-9]{5}', dir_name): - # print("Signal number incorrect") - # sys.exit(1) - - files_inside = [each for each in (each_dir.iterdir()) if (each.suffix in ['.idx', '.iqdw', '.ppdw'] and - each_dir.stem == dir_name)] - - # print("Files inside", files_inside) - if len(files_inside) != 3: - print(".idx, .iqdw or .ppdw file missing in", each_dir) - sys.exit(1) - - iqdw_file = (input_dir / each_dir / each_dir.stem).with_suffix('.iqdw') - ppdw_file = (input_dir / each_dir / each_dir.stem).with_suffix('.ppdw') - # print("PPDW file", ppdw_file) - idx_file = (input_dir / each_dir / each_dir.stem).with_suffix('.idx') - - ppdw_reader = PPDWReader(ppdw_file) - ppdw_reader.read_all_frames_left() - - database = client['ELINT'] - ts = (ppdw_reader.container.start_time() - np.datetime64('1970-01-01T00:00:00')) / np.timedelta64(1, 's') - time_tuple = datetime.utcfromtimestamp(ts).timetuple() - julian_date_string = str(time_tuple.tm_year)[2:] + str(time_tuple.tm_yday).zfill(3) - print("Julian", julian_date_string, str(ppdw_file)) - code_collection = database[input_dir.name] - year_collection = code_collection[str(time_tuple.tm_year)] - source_collection = year_collection[source] - - json_representation = ppdw_reader.container.to_json() - json_representation['SIGNALNUMBER'] = dir_name - json_representation['SOURCEFILE'] = str(ppdw_file) - source_collection.insert_one(json_representation)