{ "cells": [ { "cell_type": "code", "execution_count": 2, "source": [ "import nixio\n", "import time\n", "import numpy as np\n", "\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline \n", "%config InlineBackend.figure_formats = ['retina'] # only for users with a high resolution display " ], "outputs": [], "metadata": {} }, { "cell_type": "markdown", "source": [ "## Exercise 1\n", "\n", "1. Run the code below and check for the file size.\n", "2. Replace line 37 with ``nixfile = nixio.File.open(\"radar_trap.nix\", nixio.FileMode.Overwrite, compression=nixio.Compression.DefalteNormal)``\n", "3. File size in 1 should be about 80MB, with 2. it should be about 20MB" ], "metadata": {} }, { "cell_type": "code", "execution_count": null, "source": [ "\n", "def generate_number_plate():\n", " letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'\n", " chosen_letters = np.random.choice(list(letters), 4)\n", " number_plate = \"%s-%s-%i\" % (\"\".join(chosen_letters[:2]), \"\".join(chosen_letters[2:]), np.random.randint(999))\n", " return number_plate\n", "\n", "\n", "def get_image():\n", " imgs = glob.glob(os.path.join(\"resources\", \"radar_trap_*.png\"))\n", " img = np.random.choice(imgs)\n", " img_data = plt.imread(img)\n", " return img_data\n", "\n", "\n", "def radar_trap_data(duration=1000, car_probability=0.15, dt=0.01, speed_limit=50, speeder_probability=0.01):\n", " time = np.arange(0, duration, dt)\n", " car_times = time[np.random.random(len(time)) < (car_probability * dt)]\n", " car_speeds = speed_limit + np.random.randn(len(car_times)) * 0.05 * speed_limit\n", " indices = np.arange(len(car_times), dtype=int)\n", " np.random.shuffle(indices)\n", " speeders = indices[:int(np.round(speeder_probability * len(car_times)))]\n", " car_speeds[speeders] += 0.5 * speed_limit\n", " fines = car_speeds[speeders] * 1.5 + 15 # Euros ;)\n", " \n", " number_plates = []\n", " pictures = []\n", " for i in range(len(fines)):\n", " number_plates.append(generate_number_plate())\n", " pictures.append(get_image())\n", "\n", " return car_times, car_speeds, speeders, fines, number_plates, pictures\n", "\n", "times, speeds, speeders, fines, number_plates, pictures = radar_trap_data(duration=5000)\n", "\n", "# store data to nix\n", "nixfile = nixio.File.open(\"radar_trap.nix\", nixio.FileMode.Overwrite)\n", "block = nixfile.create_block(\"radar trap\", \"speed_measurements\")\n", "\n", "speed_array = block.create_data_array(\"car speeds\", \"nix.irregular_sampled\", data=speeds, label=\"speed\", unit=\"km/h\")\n", "speed_array.append_range_dimension(ticks=times, unit=\"s\", label=\"time\")\n", "\n", "speeder_times = times[speeders]\n", "speeder_times_array = block.create_data_array(\"speeder times\", \"nix.events\", data=speeder_times, label=\"time\", unit=\"s\")\n", "speeder_times_array.append_set_dimension()\n", "\n", "speeder_fines_array = block.create_data_array(\"speeder fines\", \"nix.data.collection\", data=fines, label=\"fines\", unit=\"EUR\")\n", "speeder_fines_array.append_set_dimension()\n", "\n", "speeder_number_plates = block.create_data_array(\"number plates\", \"nix.data.collection\", dtype=nixio.DataType.String, data=number_plates)\n", "speeder_number_plates.append_set_dimension()\n", "\n", "# rearrange the picture data into one np.array with the first dimension representing the number of positions/speeders\n", "picture_data = np.stack(pictures, axis=0)\n", "picture_array = block.create_data_array(\"pictures\", \"nix.data.collection.images\", data=picture_data)\n", "picture_array.append_set_dimension()\n", "picture_array.append_sampled_dimension(1, label=\"height\", unit=\"pixel\")\n", "picture_array.append_sampled_dimension(1, label=\"width\", unit=\"pixel\")\n", "picture_array.append_set_dimension(labels=[\"R\", \"G\", \"B\", \"A\"])\n", "\n", "speeder_tag = block.create_multi_tag(\"Speeders\", \"nix.event_detection\", positions=speeder_times_array)\n", "speeder_tag.references.append(speed_array)\n", "\n", "speeder_tag.create_feature(speeder_fines_array, nixio.LinkType.Indexed)\n", "speeder_tag.create_feature(speeder_number_plates, nixio.LinkType.Indexed)\n", "speeder_tag.create_feature(picture_array, nixio.LinkType.Indexed)\n", "\n", "nixfile.close()" ], "outputs": [], "metadata": {} }, { "cell_type": "markdown", "source": [ "## Exercise 2: Store data with calibration" ], "metadata": {} }, { "cell_type": "code", "execution_count": null, "source": [ "import matplotlib.pyplot as plt\n", "\n", "nixfile = nixio.File.open(\"data_acquisition.nix\", nixio.FileMode.ReadOnly)\n", "block = nixfile.blocks[\"session 1\"]\n", "data_array = block.data_arrays[\"measurement\"]\n", "\n", "read_data = data_array[:1000]\n", "nixfile.close()\n", "\n", "plt.plot(time[:1000] + 0.05, read_data, label=\"from file\")\n", "plt.plot(time[:1000], data[:1000], label=\"original\")\n", "plt.legend()\n" ], "outputs": [], "metadata": {} }, { "cell_type": "markdown", "source": [ "## Exercise 3: effect of chunking on write performance" ], "metadata": {} }, { "cell_type": "code", "execution_count": 13, "source": [ "\n", "def record_data(samples, channels, dt):\n", " data = np.zeros((samples, channels))\n", " t = np.arange(samples) * dt\n", " for i in range(channels):\n", " phase = i * 2 * np.pi / channels\n", " data[:, i] = np.sin(2 * np.pi * t + phase) + (np.random.randn(samples) * 0.1)\n", "\n", " return data\n", "\n", "\n", "def write_nixfile(filename, chunk_samples=1000, number_of_channels= 10, dt=0.001, chunk_count=100, compression=nixio.Compression.No):\n", " nixfile = nixio.File.open(filename, nixio.FileMode.Overwrite, compression=compression)\n", " block = nixfile.create_block(\"Session 1\", \"nix.recording_session\")\n", " data_array = block.create_data_array(\"multichannel_data\", \"nix.sampled.multichannel\", dtype=nixio.DataType.Double,\n", " shape=(chunk_samples, number_of_channels), label=\"voltage\", unit=\"mV\")\n", " data_array.append_sampled_dimension(0.001, label=\"time\", unit=\"s\")\n", " data_array.append_set_dimension(labels=[\"channel %i\" % i for i in range(number_of_channels)])\n", " \n", " total_samples = chunk_count * chunk_samples\n", " data = record_data(total_samples, number_of_channels, dt)\n", " chunks_recorded = 0\n", " t0 = time.time()\n", " while chunks_recorded < chunk_count:\n", " start_index = chunk_samples * chunks_recorded\n", " if chunks_recorded == 0:\n", " data_array.write_direct(data[start_index:start_index + chunk_samples, :])\n", " else:\n", " data_array.append(data[start_index:start_index+chunk_samples, :], axis=0)\n", " chunks_recorded += 1\n", " total_time = time.time() - t0\n", "\n", " nixfile.close()\n", " return total_time\n", "\n", "total_samples = 1000000 # 1 Mio\n", "chunk_sizes = [50, 100, 500, 1000, 5000, 10000, 50000, 100000]\n", "chunk_counts = [total_samples//cs for cs in chunk_sizes]\n", "times = []\n", "for cs, cc in zip(chunk_sizes, chunk_counts):\n", " time_needed = write_nixfile(\"chunking_test.nix\", chunk_samples=cs, chunk_count=cc)\n", " times.append(time_needed)\n", " print(cs, cc, time_needed)\n" ], "outputs": [], "metadata": {} }, { "cell_type": "code", "execution_count": null, "source": [], "outputs": [], "metadata": {} } ], "metadata": { "orig_nbformat": 4, "language_info": { "name": "python", "version": "3.8.10", "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "file_extension": ".py" }, "kernelspec": { "name": "python3", "display_name": "Python 3.8.10 64-bit" }, "interpreter": { "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" } }, "nbformat": 4, "nbformat_minor": 2 }