From dad63aa496eb33e8fc3ac2aca4721c2ba58c2172 Mon Sep 17 00:00:00 2001
From: Kacper Donat <kadet1090@gmail.com>
Date: Tue, 26 May 2020 22:10:33 +0200
Subject: [PATCH] Add get_metrics.py

---
 .gitignore         |  3 ++-
 Makefile           | 10 ++++++++++
 get_metrics.py     | 36 ++++++++++++++++++++++++++++++++++++
 prepare_dataset.py |  2 +-
 4 files changed, 49 insertions(+), 2 deletions(-)
 create mode 100644 get_metrics.py

diff --git a/.gitignore b/.gitignore
index 0fcc7c9..accea65 100644
--- a/.gitignore
+++ b/.gitignore
@@ -145,4 +145,5 @@ cython_debug/
 /dataset/processed/
 !/dataset/processed/.gitkeep
 /dataset/manifest.csv
-/eval/
\ No newline at end of file
+/eval/
+metrics.csv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 0918fbf..3ab46d2 100644
--- a/Makefile
+++ b/Makefile
@@ -3,3 +3,13 @@ image: Dockerfile
 
 raport.pdf: raport.md 
 	pandoc -f markdown-implicit_figures -V geometry:margin=1in $^ -o $@
+
+manifest: dataset/midi/*.mid dataset/wav/*.wav
+	python prepare_dataset.py
+
+dataset: manifest
+	python /opt/conda/envs/magenta/lib/python3.7/site-packages/magenta/models/onsets_frames_transcription/onsets_frames_transcription_create_tfrecords.py --csv="./dataset/manifest.csv" --output_directory="./dataset/processed" --wav_dir="./dataset/wav" --midi_dir="./dataset/midi" --expected_splits="test"
+
+test: dataset
+	onsets_frames_transcription_infer --model_dir="${MODEL_DIR}" --output_dir="./eval/" --examples_path=./dataset/processed/test.tfrecord* --hparams="use_cudnn=false" --preprocess_examples=True
+	
\ No newline at end of file
diff --git a/get_metrics.py b/get_metrics.py
new file mode 100644
index 0000000..ca20179
--- /dev/null
+++ b/get_metrics.py
@@ -0,0 +1,36 @@
+import tensorflow as tf 
+import pandas as pd 
+from glob import glob, fnmatch
+from argparse import ArgumentParser
+
+parser = ArgumentParser(description="Extract metrics from TF logs.")
+parser.add_argument('--log', '-d', dest="pattern", default="./eval/event*", help="Glob pattern for TF event logs.")
+parser.add_argument('--output', '-o', dest="output", default="./metrics.csv", help="Name of output csv file.")
+parser.add_argument('metric', default=['*'], nargs='*', type=str, help="Metrics to extract");
+
+args = parser.parse_args()
+
+
+def extract_metrics(path):
+    runlog = pd.DataFrame(columns=['metric', 'value'])
+
+    for e in tf.train.summary_iterator(path):
+        for v in e.summary.value:
+            if any([fnmatch.fnmatch(v.tag, pattern) for pattern in args.metric]):
+                runlog = runlog.append({
+                    'metric': v.tag, 
+                    'value':v.simple_value
+                }, ignore_index=True)
+
+    return runlog
+
+metrics = pd.DataFrame()
+for path in glob(args.pattern):
+    try:
+        metrics = metrics.append(extract_metrics(path))
+    except:
+        print(f'Event file corrupted: {path}, skipping...')
+
+print(metrics)
+
+metrics.to_csv(args.output)
\ No newline at end of file
diff --git a/prepare_dataset.py b/prepare_dataset.py
index 371ed01..7d92d53 100644
--- a/prepare_dataset.py
+++ b/prepare_dataset.py
@@ -20,7 +20,7 @@ if not args.no_convert:
         wav = midi.replace('.mid', '.wav').replace('/midi/', '/wav/')
 
         processed = processed + 1
-        if 'n' in args and processed > args.n:
+        if args.n is not None and processed > args.n:
             break
 
         if path.isfile(wav):