From 60e9dfa5ad6e5598e7532588a74da070b7474f9d Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 6 Apr 2026 15:59:23 +0200 Subject: [PATCH] Initial commit --- .idea/.gitignore | 8 + .idea/image-recognizer.iml | 10 + .idea/inspectionProfiles/Project_Default.xml | 30 +++ .../inspectionProfiles/profiles_settings.xml | 6 + .idea/misc.xml | 7 + .idea/modules.xml | 8 + __pycache__/main.cpython-310.pyc | Bin 0 -> 519 bytes color_plots.py | 185 ++++++++++++++++++ data/__pycache__/mnist_loader.cpython-310.pyc | Bin 0 -> 1583 bytes data/mnist_loader.py | 31 +++ main.py | 67 +++++++ neural_net/__pycache__/epoch.cpython-310.pyc | Bin 0 -> 2544 bytes neural_net/__pycache__/mnist.cpython-310.pyc | Bin 0 -> 1883 bytes .../__pycache__/neural_net.cpython-310.pyc | Bin 0 -> 3186 bytes .../__pycache__/trainer.cpython-310.pyc | Bin 0 -> 2299 bytes .../transform_layer.cpython-310.pyc | Bin 0 -> 3198 bytes .../activation_layer.cpython-310.pyc | Bin 0 -> 2991 bytes .../__pycache__/relu_layer.cpython-310.pyc | Bin 0 -> 1515 bytes .../activation_layers/activation_layer.py | 95 +++++++++ neural_net/activation_layers/relu_layer.py | 23 +++ neural_net/activation_layers/sigmoid_layer.py | 24 +++ neural_net/epoch.py | 47 +++++ .../__pycache__/activation.cpython-310.pyc | Bin 0 -> 751 bytes .../__pycache__/loss.cpython-310.pyc | Bin 0 -> 1001 bytes neural_net/functions/activation.py | 13 ++ neural_net/functions/loss.py | 27 +++ neural_net/mnist.py | 34 ++++ neural_net/neural_net.py | 127 ++++++++++++ neural_net/trainer.py | 65 ++++++ neural_net/transform_layer.py | 73 +++++++ test.py | 59 ++++++ tests/__pycache__/mnist.cpython-310.pyc | Bin 0 -> 2013 bytes tests/__pycache__/relu_layer.cpython-310.pyc | Bin 0 -> 2389 bytes .../__pycache__/sigmoid_layer.cpython-310.pyc | Bin 0 -> 2388 bytes tests/mnist.py | 78 ++++++++ tests/relu_layer.py | 154 +++++++++++++++ tests/sigmoid_layer.py | 142 ++++++++++++++ ui/__pycache__/app.cpython-310.pyc | Bin 0 -> 853 bytes ui/__pycache__/app_state.cpython-310.pyc | Bin 0 -> 1079 bytes ui/app.py | 18 ++ ui/app_state.py | 23 +++ .../__pycache__/digit_drawer.cpython-310.pyc | Bin 0 -> 2665 bytes .../label_with_refresh.cpython-310.pyc | Bin 0 -> 1336 bytes .../__pycache__/number_slider.cpython-310.pyc | Bin 0 -> 1069 bytes .../__pycache__/plot_figure.cpython-310.pyc | Bin 0 -> 1355 bytes ui/components/digit_drawer.py | 61 ++++++ ui/components/label_with_refresh.py | 21 ++ ui/components/number_slider.py | 14 ++ ui/components/plot_figure.py | 27 +++ .../__pycache__/front_page.cpython-310.pyc | Bin 0 -> 3062 bytes ui/front_page/front_page.py | 76 +++++++ .../__pycache__/gradients.cpython-310.pyc | Bin 0 -> 2305 bytes .../__pycache__/layer_weights.cpython-310.pyc | Bin 0 -> 2092 bytes .../plots/__pycache__/loss.cpython-310.pyc | Bin 0 -> 1949 bytes .../__pycache__/predictions.cpython-310.pyc | Bin 0 -> 1926 bytes ui/front_page/plots/gradients.py | 41 ++++ ui/front_page/plots/layer_weights.py | 39 ++++ ui/front_page/plots/loss.py | 40 ++++ ui/front_page/plots/predictions.py | 37 ++++ .../model_overview_section.cpython-310.pyc | Bin 0 -> 2916 bytes .../neural_net_info_widget.cpython-310.pyc | Bin 0 -> 2303 bytes .../test_model_section.cpython-310.pyc | Bin 0 -> 1983 bytes .../training_information.cpython-310.pyc | Bin 0 -> 1904 bytes .../training_section.cpython-310.pyc | Bin 0 -> 3296 bytes .../sections/model_overview_section.py | 75 +++++++ .../sections/neural_net_info_widget.py | 53 +++++ ui/front_page/sections/test_model_section.py | 42 ++++ .../sections/training_information.py | 43 ++++ ui/front_page/sections/training_section.py | 79 ++++++++ ui/icons/__pycache__/icons.cpython-310.pyc | Bin 0 -> 660 bytes ui/icons/icons.py | 14 ++ ui/icons/refresh.png | Bin 0 -> 35917 bytes .../__pycache__/plotter.cpython-310.pyc | Bin 0 -> 1248 bytes ui/plotters/gradients_plotter.py | 7 + ui/plotters/loss_plotter.py | 1 + ui/plotters/plotter.py | 30 +++ ui/plotters/predictions_plotter.py | 0 ui/plotters/weights_plotter.py | 0 .../__pycache__/training_page.cpython-310.pyc | Bin 0 -> 3632 bytes ui/training_page/training_page.py | 103 ++++++++++ ui/widgets.py | 0 .../__pycache__/utils.cpython-310.pyc | Bin 0 -> 666 bytes utils/matplotlib/utils.py | 10 + 83 files changed, 2167 insertions(+) create mode 100644 .idea/.gitignore create mode 100644 .idea/image-recognizer.iml create mode 100644 .idea/inspectionProfiles/Project_Default.xml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 __pycache__/main.cpython-310.pyc create mode 100644 color_plots.py create mode 100644 data/__pycache__/mnist_loader.cpython-310.pyc create mode 100644 data/mnist_loader.py create mode 100644 main.py create mode 100644 neural_net/__pycache__/epoch.cpython-310.pyc create mode 100644 neural_net/__pycache__/mnist.cpython-310.pyc create mode 100644 neural_net/__pycache__/neural_net.cpython-310.pyc create mode 100644 neural_net/__pycache__/trainer.cpython-310.pyc create mode 100644 neural_net/__pycache__/transform_layer.cpython-310.pyc create mode 100644 neural_net/activation_layers/__pycache__/activation_layer.cpython-310.pyc create mode 100644 neural_net/activation_layers/__pycache__/relu_layer.cpython-310.pyc create mode 100644 neural_net/activation_layers/activation_layer.py create mode 100644 neural_net/activation_layers/relu_layer.py create mode 100644 neural_net/activation_layers/sigmoid_layer.py create mode 100644 neural_net/epoch.py create mode 100644 neural_net/functions/__pycache__/activation.cpython-310.pyc create mode 100644 neural_net/functions/__pycache__/loss.cpython-310.pyc create mode 100644 neural_net/functions/activation.py create mode 100644 neural_net/functions/loss.py create mode 100644 neural_net/mnist.py create mode 100644 neural_net/neural_net.py create mode 100644 neural_net/trainer.py create mode 100644 neural_net/transform_layer.py create mode 100644 test.py create mode 100644 tests/__pycache__/mnist.cpython-310.pyc create mode 100644 tests/__pycache__/relu_layer.cpython-310.pyc create mode 100644 tests/__pycache__/sigmoid_layer.cpython-310.pyc create mode 100644 tests/mnist.py create mode 100644 tests/relu_layer.py create mode 100644 tests/sigmoid_layer.py create mode 100644 ui/__pycache__/app.cpython-310.pyc create mode 100644 ui/__pycache__/app_state.cpython-310.pyc create mode 100644 ui/app.py create mode 100644 ui/app_state.py create mode 100644 ui/components/__pycache__/digit_drawer.cpython-310.pyc create mode 100644 ui/components/__pycache__/label_with_refresh.cpython-310.pyc create mode 100644 ui/components/__pycache__/number_slider.cpython-310.pyc create mode 100644 ui/components/__pycache__/plot_figure.cpython-310.pyc create mode 100644 ui/components/digit_drawer.py create mode 100644 ui/components/label_with_refresh.py create mode 100644 ui/components/number_slider.py create mode 100644 ui/components/plot_figure.py create mode 100644 ui/front_page/__pycache__/front_page.cpython-310.pyc create mode 100644 ui/front_page/front_page.py create mode 100644 ui/front_page/plots/__pycache__/gradients.cpython-310.pyc create mode 100644 ui/front_page/plots/__pycache__/layer_weights.cpython-310.pyc create mode 100644 ui/front_page/plots/__pycache__/loss.cpython-310.pyc create mode 100644 ui/front_page/plots/__pycache__/predictions.cpython-310.pyc create mode 100644 ui/front_page/plots/gradients.py create mode 100644 ui/front_page/plots/layer_weights.py create mode 100644 ui/front_page/plots/loss.py create mode 100644 ui/front_page/plots/predictions.py create mode 100644 ui/front_page/sections/__pycache__/model_overview_section.cpython-310.pyc create mode 100644 ui/front_page/sections/__pycache__/neural_net_info_widget.cpython-310.pyc create mode 100644 ui/front_page/sections/__pycache__/test_model_section.cpython-310.pyc create mode 100644 ui/front_page/sections/__pycache__/training_information.cpython-310.pyc create mode 100644 ui/front_page/sections/__pycache__/training_section.cpython-310.pyc create mode 100644 ui/front_page/sections/model_overview_section.py create mode 100644 ui/front_page/sections/neural_net_info_widget.py create mode 100644 ui/front_page/sections/test_model_section.py create mode 100644 ui/front_page/sections/training_information.py create mode 100644 ui/front_page/sections/training_section.py create mode 100644 ui/icons/__pycache__/icons.cpython-310.pyc create mode 100644 ui/icons/icons.py create mode 100644 ui/icons/refresh.png create mode 100644 ui/plotters/__pycache__/plotter.cpython-310.pyc create mode 100644 ui/plotters/gradients_plotter.py create mode 100644 ui/plotters/loss_plotter.py create mode 100644 ui/plotters/plotter.py create mode 100644 ui/plotters/predictions_plotter.py create mode 100644 ui/plotters/weights_plotter.py create mode 100644 ui/training_page/__pycache__/training_page.cpython-310.pyc create mode 100644 ui/training_page/training_page.py create mode 100644 ui/widgets.py create mode 100644 utils/matplotlib/__pycache__/utils.cpython-310.pyc create mode 100644 utils/matplotlib/utils.py diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/image-recognizer.iml b/.idea/image-recognizer.iml new file mode 100644 index 0000000..2c80e12 --- /dev/null +++ b/.idea/image-recognizer.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..267f97c --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,30 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..95ca66a --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..18e6ba7 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/__pycache__/main.cpython-310.pyc b/__pycache__/main.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77138ac83f9d5880aae84036d6186f123b67a5d2 GIT binary patch literal 519 zcmYjO!D<^Z5S6rE?XF|5O~?o2YD(g}DNS#^IKH%}C9rIx$)cc@7D-!(drYtSf>MZ& zx#WX%D}_KlAeWpvYda8uW_Y7F^PVI#9uEnwmy@Tf4JG7X58kgB!Drm=2L?d|RivPp znPN>;M3ex%9*9T`M103+&)_ObNbr>3k|Ko!V+Ik%Aq?*iAcagMo9KcSqhbuX zNMZDw{9(5gJ93f1_zemAmCT0Es7+Qg8uD|gonJv`E42duoMQdkjD@VVa4DTMI;5^^ z5jltDNe#aA;Cv~Yss{J2psMenT>bOt@yp-!ud_8@8s`?r>r5<5VLHD!o=rlmyQaMf zOt;~n^HRCfrq{O&Wy93r@YYK0P3uwFh3Sr?OeM-4kt5M2J)7?x@SSj&ZV9E{UImj6 zvA31fVW7}RNcJ{Dyj6H0_NX7#hu07D)|zX$^lq-8vRdkTzMX!8k6>yo*I?%j&N@Z! f>Bq*1PQfSqT;lrvl#Ey&O&MpLa&|~_dPx5Ru=JM@ literal 0 HcmV?d00001 diff --git a/color_plots.py b/color_plots.py new file mode 100644 index 0000000..496584f --- /dev/null +++ b/color_plots.py @@ -0,0 +1,185 @@ +from colorspacious import cspace_converter + +import numpy as np +from matplotlib import pyplot as plt +import matplotlib as mpl + +cmaps = {} + +gradient = np.linspace(0, 1, 256) +gradient = np.vstack((gradient, gradient)) + +def plot_color_gradients(category, cmap_list): + # Create figure and adjust figure height to number of colormaps + nrows = len(cmap_list) + figh = 0.35 + 0.15 + (nrows + (nrows - 1) * 0.1) * 0.22 + fig, axs = plt.subplots(nrows=nrows + 1, figsize=(6.4, figh)) + fig.subplots_adjust(top=1 - 0.35 / figh, bottom=0.15 / figh, + left=0.2, right=0.99) + axs[0].set_title(f'{category} colormaps', fontsize=14) + + for ax, name in zip(axs, cmap_list): + ax.imshow(gradient, aspect='auto', cmap=mpl.colormaps[name]) + ax.text(-0.01, 0.5, name, va='center', ha='right', fontsize=10, + transform=ax.transAxes) + + # Turn off *all* ticks & spines, not just the ones with colormaps. + for ax in axs: + ax.set_axis_off() + + # Save colormap list for later. + cmaps[category] = cmap_list + +plot_color_gradients('Perceptually Uniform Sequential', + ['viridis', 'plasma', 'inferno', 'magma', 'cividis']) +plot_color_gradients('Sequential', + ['Greys', 'Purples', 'Blues', 'Greens', 'Oranges', 'Reds', + 'YlOrBr', 'YlOrRd', 'OrRd', 'PuRd', 'RdPu', 'BuPu', + 'GnBu', 'PuBu', 'YlGnBu', 'PuBuGn', 'BuGn', 'YlGn']) +plot_color_gradients('Sequential', + ['Greys', 'Purples', 'Blues', 'Greens', 'Oranges', 'Reds', + 'YlOrBr', 'YlOrRd', 'OrRd', 'PuRd', 'RdPu', 'BuPu', + 'GnBu', 'PuBu', 'YlGnBu', 'PuBuGn', 'BuGn', 'YlGn']) +plot_color_gradients('Sequential (2)', + ['binary', 'gist_yarg', 'gist_gray', 'gray', 'bone', + 'pink', 'spring', 'summer', 'autumn', 'winter', 'cool', + 'Wistia', 'hot', 'afmhot', 'gist_heat', 'copper']) +plot_color_gradients('Diverging', + ['PiYG', 'PRGn', 'BrBG', 'PuOr', 'RdGy', 'RdBu', 'RdYlBu', + 'RdYlGn', 'Spectral', 'coolwarm', 'bwr', 'seismic']) +plot_color_gradients('Cyclic', ['twilight', 'twilight_shifted', 'hsv']) +plot_color_gradients('Qualitative', + ['Pastel1', 'Pastel2', 'Paired', 'Accent', 'Dark2', + 'Set1', 'Set2', 'Set3', 'tab10', 'tab20', 'tab20b', + 'tab20c']) +plot_color_gradients('Miscellaneous', + ['flag', 'prism', 'ocean', 'gist_earth', 'terrain', + 'gist_stern', 'gnuplot', 'gnuplot2', 'CMRmap', + 'cubehelix', 'brg', 'gist_rainbow', 'rainbow', 'jet', + 'turbo', 'nipy_spectral', 'gist_ncar']) + +plt.show() + +mpl.rcParams.update({'font.size': 12}) + +# Number of colormap per subplot for particular cmap categories +_DSUBS = {'Perceptually Uniform Sequential': 5, 'Sequential': 6, + 'Sequential (2)': 6, 'Diverging': 6, 'Cyclic': 3, + 'Qualitative': 4, 'Miscellaneous': 6} + +# Spacing between the colormaps of a subplot +_DC = {'Perceptually Uniform Sequential': 1.4, 'Sequential': 0.7, + 'Sequential (2)': 1.4, 'Diverging': 1.4, 'Cyclic': 1.4, + 'Qualitative': 1.4, 'Miscellaneous': 1.4} + +# Indices to step through colormap +x = np.linspace(0.0, 1.0, 100) + +# Do plot +for cmap_category, cmap_list in cmaps.items(): + + # Do subplots so that colormaps have enough space. + # Default is 6 colormaps per subplot. + dsub = _DSUBS.get(cmap_category, 6) + nsubplots = int(np.ceil(len(cmap_list) / dsub)) + + # squeeze=False to handle similarly the case of a single subplot + fig, axs = plt.subplots(nrows=nsubplots, squeeze=False, + figsize=(7, 2.6*nsubplots)) + + for i, ax in enumerate(axs.flat): + + locs = [] # locations for text labels + + for j, cmap in enumerate(cmap_list[i*dsub:(i+1)*dsub]): + + # Get RGB values for colormap and convert the colormap in + # CAM02-UCS colorspace. lab[0, :, 0] is the lightness. + rgb = mpl.colormaps[cmap](x)[np.newaxis, :, :3] + lab = cspace_converter("sRGB1", "CAM02-UCS")(rgb) + + # Plot colormap L values. Do separately for each category + # so each plot can be pretty. To make scatter markers change + # color along plot: + # https://stackoverflow.com/q/8202605/ + + if cmap_category == 'Sequential': + # These colormaps all start at high lightness, but we want them + # reversed to look nice in the plot, so reverse the order. + y_ = lab[0, ::-1, 0] + c_ = x[::-1] + else: + y_ = lab[0, :, 0] + c_ = x + + dc = _DC.get(cmap_category, 1.4) # cmaps horizontal spacing + ax.scatter(x + j*dc, y_, c=c_, cmap=cmap, s=300, linewidths=0.0) + + # Store locations for colormap labels + if cmap_category in ('Perceptually Uniform Sequential', + 'Sequential'): + locs.append(x[-1] + j*dc) + elif cmap_category in ('Diverging', 'Qualitative', 'Cyclic', + 'Miscellaneous', 'Sequential (2)'): + locs.append(x[int(x.size/2.)] + j*dc) + + # Set up the axis limits: + # * the 1st subplot is used as a reference for the x-axis limits + # * lightness values goes from 0 to 100 (y-axis limits) + ax.set_xlim(axs[0, 0].get_xlim()) + ax.set_ylim(0.0, 100.0) + + # Set up labels for colormaps + ax.xaxis.set_ticks_position('top') + ticker = mpl.ticker.FixedLocator(locs) + ax.xaxis.set_major_locator(ticker) + formatter = mpl.ticker.FixedFormatter(cmap_list[i*dsub:(i+1)*dsub]) + ax.xaxis.set_major_formatter(formatter) + ax.xaxis.set_tick_params(rotation=50) + ax.set_ylabel('Lightness $L^*$', fontsize=12) + + ax.set_xlabel(cmap_category + ' colormaps', fontsize=14) + + fig.tight_layout(h_pad=0.0, pad=1.5) + plt.show() + +mpl.rcParams.update({'font.size': 14}) + +# Indices to step through colormap. +x = np.linspace(0.0, 1.0, 100) + +gradient = np.linspace(0, 1, 256) +gradient = np.vstack((gradient, gradient)) + + +def plot_color_gradients(cmap_category, cmap_list): + fig, axs = plt.subplots(nrows=len(cmap_list), ncols=2) + fig.subplots_adjust(top=0.95, bottom=0.01, left=0.2, right=0.99, + wspace=0.05) + fig.suptitle(cmap_category + ' colormaps', fontsize=14, y=1.0, x=0.6) + + for ax, name in zip(axs, cmap_list): + + # Get RGB values for colormap. + rgb = mpl.colormaps[name](x)[np.newaxis, :, :3] + + # Get colormap in CAM02-UCS colorspace. We want the lightness. + lab = cspace_converter("sRGB1", "CAM02-UCS")(rgb) + L = lab[0, :, 0] + L = np.float32(np.vstack((L, L, L))) + + ax[0].imshow(gradient, aspect='auto', cmap=mpl.colormaps[name]) + ax[1].imshow(L, aspect='auto', cmap='binary_r', vmin=0., vmax=100.) + pos = list(ax[0].get_position().bounds) + x_text = pos[0] - 0.01 + y_text = pos[1] + pos[3]/2. + fig.text(x_text, y_text, name, va='center', ha='right', fontsize=10) + + # Turn off *all* ticks & spines, not just the ones with colormaps. + for ax in axs.flat: + ax.set_axis_off() + + plt.show() + +for cmap_category, cmap_list in cmaps.items(): + plot_color_gradients(cmap_category, cmap_list) diff --git a/data/__pycache__/mnist_loader.cpython-310.pyc b/data/__pycache__/mnist_loader.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cddaf7aaa13a57a4f52859408f263c5561163eef GIT binary patch literal 1583 zcmZ`(&u<(x6t+F~>}*KVAEaqfL-~ z3W;V<8>uJ6g+E|(?7w7j|Refj6>{tHgX zU)Wjvz(>A;Y99h9qG(RySxaLEeU=9U9&<{*BPvk*geb20IEaO!&qzm1VHW9isBb?l zmCnCST#}+|@eAf1`#Bhz#8i=(y(E3AsA4Bk9Lxg8TnQDPkXXDVoWvo7Z~4`yz1?R& zT*MX&fzghjVto5OfFolvp*5MXnx0}C)U0V)9nAaul%B$29Cu<8*5sh2=m+GKP9g;( z$CSd+Vcno-D8%cq=>{|gIqI~%u)|6lA4!=NnUm75$$jl)Hc0lhm3>nVWTt+dd-G&Z z=L=p@nXGW06_po>G0BmC&uI&@i)!esZy9ZWPAc6Io?D&w<7@pwI+J9@d_;Wxjl(4d z*0u2}dM;wjreJ?Q;1#GMWtu0}O8JVMJ?>VfJkY7L-CQT8$cp`Lv-vNKPRsowJJzPF zK*8>y$RJE!CQ6&_>d1sZ05WY?0YVzh3;+9GczpY9g|^SxNg0ei|8O$aCs6Gl0FKlk z|Cs&$)rdJdVT#pk0*L_fu;*+PjQAB^F$x_&5H)jrf>x^$4kT6LH#Tb3tx3D)wWz~2 z(g!(Jc$bV?3jBA-kA#7=9LhMLAn_w&pB#eRSbl=q$R71PGka&j&hG9McY8*6fJ33& zQKhH!HvJ`Rv#E3;<%0uf#k(Rmj9HlWgVE7DfqSXRGSEI}_ zU5B#4)@jJLULS8>R_2OqeM_ZtnH!Z9M17;u<~vn-w;rMvSFrxs|5y)u=6yWM8p17v zb%fghowm7)c5@3qMC%>`Qfxj#Kz7Z?2%i9S!iG)rDLOV0wh-YPDhN@be5W%*h_Vp!$UN{~S0r@Z-z@=;w?N*_OCdp-?-S&lP@a{Ep7M5rZ Wk4yOI>)CF1FrDY?L~NO@vHt*k`FZL9 literal 0 HcmV?d00001 diff --git a/data/mnist_loader.py b/data/mnist_loader.py new file mode 100644 index 0000000..ff42fb3 --- /dev/null +++ b/data/mnist_loader.py @@ -0,0 +1,31 @@ +import struct + +import numpy as np + +from neural_net.neural_net import ModelData + +class MNISTModelData(ModelData): + def __init__(self, fn_train_inputs, fn_train_targets, fn_test_inputs, fn_test_targets): + super().__init__( + self._get_images_from_idx(fn_train_inputs), + self._get_labels_from_idx(fn_train_targets), + self._get_images_from_idx(fn_test_inputs), + self._get_labels_from_idx(fn_test_targets) + ) + print(np.array(self.test_inputs[0]).reshape((28, 28))) + + def _get_images_from_idx(self, file): + with open(file, 'rb') as f: + magic, size = struct.unpack(">II", f.read(8)) + nrows, ncols = struct.unpack(">II", f.read(8)) + + data = np.fromfile(f, dtype=np.dtype(np.uint8).newbyteorder('>')) + data = data.reshape((size, nrows * ncols)) / 255 + + return 1 - data + + def _get_labels_from_idx(self, file): + with open(file, 'rb') as f: + magic, size = struct.unpack(">II", f.read(8)) + data = np.fromfile(f, dtype=np.dtype(np.uint8).newbyteorder('>')) + return data diff --git a/main.py b/main.py new file mode 100644 index 0000000..321be7b --- /dev/null +++ b/main.py @@ -0,0 +1,67 @@ +from ui.app import App + +if __name__ == '__main__': + app = App() + app.mainloop() + +# import numpy as np +# from matplotlib import pyplot as plt +# +# import matplotlib +# +# matplotlib.use("TkAgg") +# np.random.seed(0) +# +# from utils.mnist import MNISTNeuralNet +# +# # Set the precision to 3 decimal places +# np.set_printoptions(precision=8, suppress=True) +# +# from utils.load_mnist import get_test_images, get_test_labels, get_train_images, get_train_labels +# +# train_images = get_train_images() +# train_labels = get_train_labels() +# +# mnist_neural_net = MNISTNeuralNet() +# losses = mnist_neural_net.train(train_images, train_labels, 0.0001, 100) + +# test_images = get_test_images() +# test_labels = get_test_labels() +# results = mnist_neural_net.forward(test_images) +# predictions = results.argmax(axis=1) +# +# correct = predictions == test_labels +# incorrect = predictions != test_labels +# accuracy = mnist_neural_net.accuracy(results, test_labels) +# # Create figure and axes +# fig, ax = plt.subplots(figsize=(10, 5)) +# +# ax.hist(test_labels[correct], bins=np.arange(11)-0.5, alpha=0.5, label="Correct", color="green") +# ax.hist(test_labels[incorrect], bins=np.arange(11)-0.5, alpha=0.5, label="Incorrect", color="red") +# ax.set_xticks(range(10)) +# ax.set_xlabel("True Label") +# ax.set_ylabel("Count") +# ax.set_title(f"Accuracy {accuracy}") +# ax.legend() +# +# fig.show() + +# while True: +# plt.pause(0.1) + +#################### +## Draw image ## +#################### +# Create a figure and axes +# fig, ax = plt.subplots() + +# Initial matrix displayed +# initial_data = np.array(images[0]) +# mat = ax.matshow(initial_data.reshape(28, 28), cmap='bwr') +# fig.show() + +# Redraw the canvas +# fig.canvas.draw() +# fig.canvas.flush_events() +# +# plt.pause(20) diff --git a/neural_net/__pycache__/epoch.cpython-310.pyc b/neural_net/__pycache__/epoch.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b573df04e39c62475eedd11d3aab1eedaa1a25bd GIT binary patch literal 2544 zcmb_eOK;mo5Z+xrM9Y%wxOq5D+Nx>#VB1(hpFj`iC=Zk?`6dhf-*Jm|6M~CFfVhuY zJOogz$0h3t$)$M7dX99Yi!XRdA74+_WPqivLOyabr=35%cvARYr06Bp2#cz|x70eYn`MOmAODNgOWQ|BRjy8glk z?i~c>(!`$#On>5CVLywjY~n0ZQ|FL#_WYXVRJU;1!iIFfyD1q&b-R6tg7N3Cn&a~* zY#C3H@W#!CMPUNDr{gjm=BAmcSm$ZpkFcOjeLF6bov291%G4~b3ghWG?;29VcJ4%K% z;}$A=rYRJRpXQ@IZ2MWft+GOsxi#6@@brl3OTgZI-x=xQu1db^1BH$fEP(K3pNYq1XC^ZD;G-#Dh~eB83$-k4oT$dI!^I7OIs zdkaX(4%i_%7jd@9j(9f|#x2tUELxPYE+fK^xW0h1R)q>&25CHd;v&$3c((X>xq8~J zC$u&jgrB_oB?ud62)YAqUl*lw;8+Ynw@cGoC#ifJ7NWZq*7Pc_nU%gOqd3c=V`ubM zO7f?Yvju$>$9e%=ZAI7bg47;BD6A5uJ~!`Kh5X8~S8%!r0DNA@Z;g)|m0s2MTLvYU zkU{^t4giC4%;S`WBFK^{Kg195?Q~CFCW}z$Y|`a507{6b-U~kAyX+V7Tx?c054l+$ zX(iKyLccf?Mr__0)BztbMC$|vnk`|+$OV`bT{JU~gl z(+*wZjdYrqw)AQeOE(Fe=RV{PPDYCQ$W0uwxxL0$@w>>!7b-t4oau&js<(MOP*G&+ zQ8XCJy^Qci6g}UIvua0EozRr?+C~MHHN8QgL*Od{^E|N_l}$|8(f)=oV2d%&f z{6GYv;n>{-8>a?cxsO>;v3>hb0d-E@z*a(rm<__|qV*SvS+#gHX;gbOXS8_RfF1IJ zrj`P$E>(Rr-y3Kfh&I7)*uAO;d=E48YjNfeHox0 z$WVm`E!iSD3}htZ(;$gv)|HvHEmzSR%Z^+-Wyy;BeqC5yJ^>-4}RGF z={?SjI}}%f^cGGZr|zRL?1UX~;}It5pZKT+(w8A>K2jN;cq3m%Z#mISR1Sb#sl`=bJm21-1i`@;E_)nkdj<+VL>(TGa7f<6u>s{c0eeSy+I)KN20g9@A|qg=jQ(E;@Q{4Z zE+*oxUHphPZ(O`Vj?+aoZPS^LQA{8I2ZE)8*OHfFr0a+V{RD;N$NY#Lumcuuh=(!^PaL}{xUq`XWls_u0Q?qg)$9PUD4fdN>O1(2`Tmp)zt;; zayX(y%QItJ_{^m^oCRQ8c{Of~wyPjsr3B{EeM*hm)X z6_QcV>cBf3ts?Qi?BzZC)PA*8TsN^BON<0|=V(MI}Ma<0wSReIID`hfo@ zu5{P5k_StthfRg&>q_@2t6k**@nP>T9a^Xm(%!>`p>IG`dRRj_yMa2QgSZseqG;o< VuE_hEI<2N0#NfP5jsfy5?_Z7@z9j$v literal 0 HcmV?d00001 diff --git a/neural_net/__pycache__/neural_net.cpython-310.pyc b/neural_net/__pycache__/neural_net.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..691e8bbe38e813cc4a7f7b7cf384a591daca0b64 GIT binary patch literal 3186 zcmZ`*OK%)S5T2g*?#`~)&dYH`fC!K_AvTa8E)j}E0++Cta6qG=VLaVlk2Cvl&#V($ zvnY~%;KH37_A$SOe=t{0IdkIzU-j(lD`8e$+g;t&U0wB6RS%oZx`xmCqt^R#UDN)e z!Qy9N@DMHi5rk`;C0d{Pj8R`t^uFO6s&6EwZ-HkfcHi+G#j_H(U-N5B`%dFFcg{5K zoa%lZBbV1Os$taN##60Re}mn%4l7#WZkow3%K9SPA98Ta$AfXdi;nepG>rDC^CUbL zG9rn^PsiXPT6!5oXg=ebuXD!rGtD>DDP~doe#k}gD9pmh!m8?{K8b&Wmfiy4EY|`p zC(3diRL>1iBR4@!u0dtJ`)$YaYjGMR!;lMJtU>R15D$7mJQ$6$)Sokwa91R0(a1!a zO=qWra;9SyW-5|B>ELX?J=-V?Wlv^I&HrhX*3c(w#(vl8)br z`(aPql_DDU2JwlII|DJ6VG;~Pw$R-k9T#o@ZQ?8lXs?ujV6*a9XD1soTWwd%x*d~b z24PU%+OI1ZC?MmDjN(rJ-vEx6Vyq2=2psC{cxd%TJN8gKD6{MSW;w}p4mgvYNJwXU*cS7e2q5D3$VU{ih6r>0_+ zXpj+?G_<5hMONADI>c6$*#hr#G-b0olrjH3S(~e(`hj(!KeHxmQk&E#jft1nxHe|;HjHiOwF~6X zyskL+aub}o;xrWJL2iN5P#jNj9_KbV9&rxWk&MjT%I!U~GK5pOQXB!0NJM55r&&=G z$eTijnJB&J?uOBGa--j@GDSeMg&mGYV!%61DUlXN%wI}!iSI4#QJYF*iBn11rOc}H2(MqFr5}Kp zKoR}6^iAgKb>_mERnuR2`eGk?2(@G$(on&vAAN>;rH*yl5z! zTRBrnU*3WQBB6|o{V)x)Ov?AcEN6_;ij6Qyg6Xx??N{zptRb-n3yXfn4k_hGiWG_! z{S~6+`W&Jw>zBZ)A`G$$tUsWWYv+h?#4{rMfKd_rjZI8$Oe{baIS*iM0j&1K$sN=; z`?~gAm+$8mXI}x{0GO2lVyEl|r7qOG0|2dKcHdB_QcVtD#;H^TOH!NzM6%ZpU&{Ar zU?gH7uh3Gr%wpHnlBx>I@I^399e!gZ1&^aF9u89f8W0;qLy6Lr1q&1ML&*E}IcL?F zmmv`*(KrE~gDQpgAe54u%FNS-OEg@k!>b8gpni7pwJurkX8gqH>5YUjuS=P>d` z`HvKOLC=M$$;u=(HIb>(ygnEeRw!k7Ea{yx&8d2{*rE0fSMW~KeF00ZdQ>d*9hB7`q zfl5Uv3JYh7wqGzvX+IR%h1xZp?RM5>iv-t*w29E$rF;NUU`Q%Fk`kEwhzJ!#xkH5Z zly`}26Cr!bdqnONnQNnZq>{w*T>BU;r9EA(&0Nd1UDLheI&NLJHkod0GS5)!I$mLf zyOBDjFsT$s@_1pPH#%15`t8B&UAV1sy(Ap?TMKNwLXrEHlS;{6(-exQKg|c1 z5i|9koT)hN$->ls&V@|R;vyd4uKNveRCkU+k)%%r=`%s4BbaoBBR%0te;5c)__uDq zhBHGZY9fG9VAq(aix5V2u_qe0q#uey(Gu-j+TRl$u@5s1(UncGdLj;BUrRIr?~&=; z;d`m#B$LX7nMKSqfE-Q`G8cf^G*cO?EXhV(#f3ECMO+LnxlZPC;N9~Dbq4(fbp08K zl9q&%l7XZGbv7a(ozem6mM%z7kaO~e^?rza<%5Z;fiPjBxte8orD;lxC%;T|k*jNk z@ltgVk!M^j)8W=^yY_~t7Hs+Z>(i;q$8u2U(~S$K$s``hFO(eQqYOMyr(35_iz*N& z(`yrOo@7bEIik`H5I)`fLOOr?ICv+U{R0farlGzG$vY%p6%y+-?M2i=j+kn~QUxDz zTkxt4n%UQ~;2Z2V{J7yAo(U+P@1WS({=ab4KHk&?0WfSp*Dz485Lhf0Mhnl~KD#wN zfPM3i--}?ob3kF&rXgw!rzOajpcJG4T8!BZN@om&0O^7Bq1SG_1nAXt9XWQthtW#p(mI8J>C0#c*Mr;%%kW!m`~J|6<7 z^<}>f6P2ot(c}^7JYB@9_y|_?5eT0$>N7?IH(&u}e+3PnHt76l$75^u;KlLv@0`Wp zp_3}8Tt?v76($Cd5YfY*&4q`E)H7H@!f2$APebbdJ3oBS{rS{7^nmnOF|agQ*8L7j zNtXlzSJE*(BfoyWfC4pUOQ&RGcSTFak%4UK!`OpfTQPVHoC%O$cmMe!$XfdU}4#9<6x6z{Mwf7VqZ(FMK4PGH3I<<6*T|WGhO9%o^oTWRi#FH?v_t2g z?7aSNPWO6I8w+_BPbB9i>eD`+`^6NW*z>)O>`Dvzyfhxewp a>K3K7f8Vtx_@?skTTDEgEDZvobN&N?IU6JZ literal 0 HcmV?d00001 diff --git a/neural_net/__pycache__/transform_layer.cpython-310.pyc b/neural_net/__pycache__/transform_layer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..82a9d98034147ed2d4ab51268176474dbdb72a04 GIT binary patch literal 3198 zcmb_e&2Jk;6rb4-d+jF1v}sByAFBvx4XKl+BA`;Lq6#6T;!7(ITUD#|&N$v=Kg`T- zi7ofiRO*TUp*i}*jX!3uoO;d$AtZQj*6YMh(i4-+)4ZASd-LA>es3pPUak{pf4ums zw^bwLPgG7e1C;yF<$CB8$sml@SB4B$xa-MxY&q!CcMy%dAb1^ie)bm zksD$Z?ZGg&TsMe=)OFE=MB^2Te|0))s8HMG1|MTA8;;AEkMLJ1T<`&XK_^;gbp3HZ zkhbh6S;*`zx07dFh=4I3+uflpv#uX{QgS(=`K3;h{um8H9`QKk?0X>+G0_CziR2ZP z7pX!m#08wLLIA&df-Qk^glDj{(Q57k+T{aw-Cl$+}hz5#xKuq+)QI$P(8Du_R zF0<$gFXfAlt*+R)mt6Ok%nOSZfmspgR$M~y9*Xx-AYe`iOr^lwAiN|-WD$#Ml{(dQ z>eFF-DBtQ{IyN5G2K5g);7OUq4}4FG4Dfa%35)&Pru1^~61>nhy2 zZjlC+0o;8Ry2O;N($SUqBOQZ(@n%tffIq}Tj>#^ZMPS8i(SopObRpfj#kllE(B)Vp zl*h_xkL8|o>9zo3@$P9)>h3=-omh0Vx)9cRL_sj9sE{LLsFB5A&zweAM0EKKj|j9i z=qjwn>9EeF$SjeVJ-kwt#=_AVNmO9|v~*<9(aJ(-iY=|0xCVO!CR%)mLh<4g9L@70 zU!KCk=V4evLvaP85eg&-FpN*45sNHQOUJ*auAt|`gIbNd;5trF3mZF+KR z-TqSj-(I)3pW5Hvv!86XH(_eCy>&v~QsdiCXFt3%@Pro?R^=T#9<)6nyy1;rn0RUH zc$?s9CgSrqz~e!eS#&OqZepAm^ip-SItyv;do!t=i8ci^gzAYppzc7WA1~?#^dl@F z2&$$baL$K5*D%@G#AhG?Doc-&Uiy=_|8}4Vw2jrVjX%YE7B&iXfO!ynh4Y_-7+pST z`t0jRu*|B&Lr_wv$_fJ7A&}&dKG#yRtB=*&is`@U&vn>gjNzrXryWwU2AT=L1@P`x z7`>`s0~K9Y@G(dK%x-R$>(lUAx4T(tuMINU=W@;N2RnT(>}Ot>aoC*LEQx{`zTk+v znLT(ab-iv72IM4?& vaTX1RVha}$&Z=%Iv={JHcsv%!kKDRjym0Pe?5fZ#!Aq~9H7_-rD~v|}f!n>JDS=r%3TmC^(d2dEHGD%499BvLNEtTvvpv&P;vGwY_Y z91b+17x)VhQgY0{G*?dCkhmaDyf?d!9Vdmc=K0N=H#6_e{N^{C)$3IPMgRDB`&FHg z-?1@UY-oG|Rec146Ha@iPn(pY&U&nA!N1kB`%crL!bq9nuVIS zaJb986Y`QaUC@1AF}laA!tYkvj4$xoiQTO7I$t~?%>};1m*Kw#zBfvb2ETb?!}vPb zR!Z9~e%sgE$)o&&FDV2ptmx%IT=#9IkuryO+lV6oDoXKjv zlXNi5m3FgX4vMBDUGK=4CnC+GRxcT7?@%P218`~JWJ%mhMk1Qn-2Eh0Lg^bbeT6A> zwXH<2t(_$0;)Sl5g(9Bxb+uRvb-4^N%La&Zsw)w!7%LS;=j5Mf+XI<(MJrd^JrT<^ zNjuv~KkkT!Qna#83Q@@IR19U@i&Bwq1K`S2i+YGQb;YtZIFcT${KAR5$#Uyaph z4gP)0r#|y(6)dB*IS_13wz5TG;H03Avps>T?t>VUF8S2}piAK&Akby-4+q?};NSku zha+=#M=W@t>5EY{zyz{XZ3d%SFwGzgZHe>B+bC91R6v|lxq>>zUWhk9(>@ofC6j&e zCbSd-ai3mIVB&No0ZalD_zQ7_m^!Ds#q4i^VQw*b18jHD8`rVfXk&h(W`IzFhu65a4ZQ~aaWN1ovu$kdfM*>holR6x!~HkKPO%HUAbXL1P|+ET+A8Vx-II_;JFrR$e6ZhE+h5g)-5 z2^(7Od(+1TZalo8pcS#sW7gk?gGA6igk=>%zE-{OG}wyM@EWvn4QWmWTwwsblLouU zXNX<%(!o1AdDjdpzy(6Ok;nn^8gkGMoVv)rhHi!AECOxIzz? zDUbq$+32f_lX^TD^o|T}r$-dDhbcTbSsFapOFI25;hVuXV&~h zOyn|q%X?tkgIW-(Gg_aIf4Zpl{x1Q!4KqRTPcDLglJ((PFRtAbR(N;tB2MSNNa$kZ z<a{5Pei-+P9`jbq zw{d=i%M28df#m`Ud~@X{3dE$@ONmkGdig2Q61`p&&FuLa`r)%_KAVt36+W9jS)~37 zv#J#9vgNz}2D2NKx$D$;hMukF5JEr%2y|>guBAW-gb+=1MG(z4*0S~;d)mjv-UV{HE|H+; z4D2BZ<1RjVIInLtg#Ir$O@<*g>#vh4{Tx?m zS?tG0QiaT{E1A_izFar=?ShcXI)0X|y#()R3*tl3Lr~IUi735l+?l&`@5}>c=$vpS2;9Vmk$doV|3l~)PuEkaOu%_sq!s6;qw9~=(Nu=M zakb1w#!Cw!znX4ZOzVn^G&jAnt}g7(7nzPfRoeI;(^yNrpr*@tlEqqcz94^xgQ+Sf zGO6?+ld&q&VmwImcr2eOnUv!qok=w)WUbt-UurQkZ6dX(H2G zv`1%~m!#}nB#}_qyb!cK7;j+cr=Th_p{Hz4CJqeuj8){+Lvkk04mqP+1ZeWlX&Qq& z%B!$rSTQxeiVIQZk%!+xt-zLA1qD1%*eig1ps= z-8|<8cv+0wbhdLN#>UTdnwroFCUhZa<3Oyx@zp9ar^nEN3kuDruv{0cp=$^1t8jYD zV!#}J<5dBjZ7p#u9sV0vJ;0kEf`W)hoWcgV8tTsSCcf-+%h6+SpY7fhv*igbH4Im< zjfyyz&_+GZ^HS6q+N+!&)^XPC*a}v65yO2{&FjQU#Ybuv7oLDJ{qr75WR`K>Lc5MG zSWyaV8oNPb9R*~K25#VPIQJa869&dB>Kr--14Pf3EwS5T4y_+Ndp+Qcx5hprGajR0P4J7jIr9bk`b9Her%fD?NJfG3>Fgl&dF? z;?0vY8>|KE!p`j9?DsQir?U<;Ud}&mjs?IQZl-(ahTv3V0P-OkpOeLS7(07|2T6tZF6U zW0aK1TvV;JEV8o5=YUOm@O2(!nBJLTkq5B}(MEO@L`gU@2WW=r$VN{F18YhQ<7;b* z;HI?0B1&x@gp|i2uMe|H)kPDR*S}|qKL2FJZGk?eKioY!bliKiVD3|LhZiAUHz8*SeDI~OeAQOp DP}h@5 literal 0 HcmV?d00001 diff --git a/neural_net/functions/__pycache__/loss.cpython-310.pyc b/neural_net/functions/__pycache__/loss.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f74d26552258f92911c493827803ca5b8d07fbf GIT binary patch literal 1001 zcmZuvJ8u&~5Z=9q<2VliibNtF+SXDMCx9p{9uf3uh$tK(Ykj*uUpVg`b9+Wc=K_&} ze}L#n$QmJG2}0??vLT;niDbe% z^Jk=_C!X-P$jnE-UlJ3P{ySRIp|?fO=n9zy=nu=_jF$clf`tM;poFv-iY`o6n7qr5 z-O;*^+1~EmJKKo+4)^Y{7uyTb`?bH$!Ke$?XgI{u@i7eV&tu(;10Vk_rm|fQz^7gI zTt-F%(?iD7l$nua8Ay>tMhdj_fDe*1F^SCC#AW8L)|~H8vH$UDC3i;JT;5|KkvW&j zBr0}=cw`EmvI$QMxj-AGN!Dcpty9U>HhVpi*cULPnT@!+_1G3WtUCi~3Q%2lh_8Oz zdHeChtB3KI^Is_69|O*vv>TPDvMR7_RhW->CM!SHaTNkrG1_@CZim&<{NGed^LK*X zi;9kW>zLe6lb140Mp}!{1XeKe^PqzhJHavK9qb$wKD3jJc`SE8Mmko>8wnjH3*c#A zNz)k?Dl&;yn6C4DH=9=1B0#H;Ljs*m`_^r1=61(})pf7sZPIlLYj})2giEnW0jgWl zn`qPvifc*j3YLg7uf)~ewCm>`Ubr#%U^yyZP9@meZ6BKWfn|hVW7;=RNcXZDR&DaF z#SVppWWr4{k*?Y6n37wn4X^3J6*SuPs2x 0, 1, 0) + +def sigmoid_activation(outputs): + return 1 / (1 + np.exp(-outputs)) + +def sigmoid_derivative_activation(outputs): + return outputs * (1 - outputs) diff --git a/neural_net/functions/loss.py b/neural_net/functions/loss.py new file mode 100644 index 0000000..6e53d76 --- /dev/null +++ b/neural_net/functions/loss.py @@ -0,0 +1,27 @@ +import numpy as np + +def cross_entropy_loss(outputs, targets, clip=True): + """ + outputs: [ + [ 0.32, 0.12, 0.04 ], + [ 0.62, 0.02, 0.14 ] + ] + targets: [ 2, 1 ] + :param outputs: np.array: Vector of all the predicted probabilities vectors + :param targets: np.array: Vector of one-hot vectors representing the actual values + :param clip: boolean, whether to clip the output probabilities + :return: + """ + if clip: + # Clipping the predictions for numerical stability + outputs = np.clip(outputs, 1e-12, 1 - 1e-12) + # Calculate cross-entropy loss and average over batch size + m = targets.shape[0] + log_likelihood = -np.log(outputs[range(m), targets]) + return np.sum(log_likelihood) / m # Average loss + +def cross_entropy_derivative_loss(outputs, targets): + # One-hot encode the labels + y_true = np.eye(outputs.shape[1])[targets] + # Derivative of cross-entropy with respect to softmax inputs + return outputs - y_true diff --git a/neural_net/mnist.py b/neural_net/mnist.py new file mode 100644 index 0000000..19466e2 --- /dev/null +++ b/neural_net/mnist.py @@ -0,0 +1,34 @@ +import numpy as np + +from neural_net.activation_layers.relu_layer import ReluLayer +from neural_net.functions.loss import cross_entropy_loss, cross_entropy_derivative_loss +from neural_net.neural_net import NeuralNet +from neural_net.transform_layer import SoftMaxLayer + +class MNISTNeuralNet(NeuralNet): + def __init__(self): + super().__init__(layers=[ + ReluLayer(0, 784, 121), + ReluLayer(1, 121, 10), + SoftMaxLayer(2, 10) + ]) + + def backward(self, dL_dout, epoch): + return super().backward(dL_dout, epoch) + + def loss(self, y_pred: np.array, y_actual: np.array): + return cross_entropy_loss(y_pred, y_actual) + + def loss_derivative(self, y_pred: np.array, targets: np.array): + return cross_entropy_derivative_loss(y_pred, targets) + + def describe(self): + """Return a human-readable string of the model architecture.""" + architecture_info = "" + for layer in self.layers: + architecture_info += f"{layer.describe()}\n" + return architecture_info.strip() + + def predict(self, inputs): + raw_outputs = super().predict(inputs) + return raw_outputs, raw_outputs.argmax(axis=1) diff --git a/neural_net/neural_net.py b/neural_net/neural_net.py new file mode 100644 index 0000000..84e7b8c --- /dev/null +++ b/neural_net/neural_net.py @@ -0,0 +1,127 @@ +from abc import abstractmethod +from enum import Enum + +import numpy as np + +from neural_net.epoch import Epoch +from neural_net.transform_layer import Layer + + +class ModelData: + def __init__(self, training_inputs, training_targets, test_inputs, test_targets): + self.is_loaded = False + self.training_inputs = training_inputs + self.training_labels = training_targets + self.test_inputs = test_inputs + self.test_labels = test_targets + + +# class TrainingSession: +# def __init__(self, training_data: ModelData, learning_rate: float, nr_epochs: int, batch_size: int = 1000): +# self.training_data = training_data +# self.learning_rate = learning_rate +# self.nr_epochs = nr_epochs +# self.batch_size = batch_size +# self.epochs: [Epoch] = [] +# for i in range(self.nr_epochs): +# self.epochs.append( +# Epoch(i, self.training_data.training_inputs, self.training_data.training_labels, self.batch_size)) +# +# def get_total_training_duration(self): +# duration = 0.0 +# for epoch in self.epochs: +# duration += epoch.duration +# return duration + + +class NeuralNet: + def __init__(self, layers: [Layer]): + self.layers = layers + self.last_loss = None + self.last_accuracy = None + + def forward(self, inputs): + outputs = inputs + for layer in self.layers: + outputs = layer.forward(outputs) + return outputs + + def reset(self): + for layer in self.layers: + layer.reset() + + def backward(self, dL_dout, epoch): + layer_dl_gradients = [] + layer_dl_bias = [] + layer_weights = [] + layer_biases = [] + + for idx, layer in reversed(list(enumerate(self.layers))): + dL_dout, dl_gradients, dl_biases, weights, biases = layer.backward(dL_dout, epoch.learning_rate) + + if dl_gradients is not None: + layer_dl_gradients.append(dl_gradients) + if dl_biases is not None: + layer_dl_bias.append(dl_biases) + if weights is not None: + layer_weights.append(weights) + if biases is not None: + layer_biases.append(biases) + + return layer_dl_gradients, layer_dl_bias, layer_weights, layer_biases + + # def train(self, training_run: TrainingRun): + # self.training_runs.append(training_run) + # + # for epoch in training_run.epochs: + # epoch.start() + # + # for batch in epoch.batches: + # batch.predictions = self.forward(batch.inputs) + # dL_dout = self.loss_derivative(batch.predictions, batch.labels) + # + # layer_dl_gradients, layer_dl_biases, layer_weights, layer_biases = self.backward(dL_dout, training_run.learning_rate, epoch) + # epoch.layer_dl_gradients.append(layer_dl_gradients) + # epoch.layer_dl_biases.append(layer_dl_biases) + # + # epoch.finish() + # epoch.loss = self.loss(epoch.all_predictions(), epoch.all_labels()) + # + # if training_run.epoch_callback is not None: + # training_run.epoch_callback(training_run, epoch) + # + # self.recalculate_loss(training_run.training_data.test_inputs, training_run.training_data.test_labels) + # self.recalculate_loss(training_run.training_data.test_inputs, training_run.training_data.test_labels) + + def get_all_weights(self): + all_weights = [] + for layer in self.layers: + if hasattr(layer, 'weights'): + all_weights.append(layer.weights) + return all_weights + + def recalculate_accuracy(self, inputs, labels): + raw_outputs = self.forward(inputs) + predictions = raw_outputs.argmax(axis=1) + num_correct_predictions = 0 + for idx, prediction in enumerate(predictions): + if prediction == labels[idx]: + num_correct_predictions += 1 + self.last_accuracy = num_correct_predictions / len(predictions) + return self.last_accuracy + + def recalculate_loss(self, inputs, labels): + raw_outputs = self.forward(inputs) + self.last_loss = self.loss(np.array(raw_outputs), np.array(labels)) + return self.last_loss + + @abstractmethod + def loss(self, outputs: np.array, labels: np.array): + pass + + @abstractmethod + def loss_derivative(self, outputs: np.array, labels: np.array): + pass + + def predict(self, inputs): + return self.forward(inputs) diff --git a/neural_net/trainer.py b/neural_net/trainer.py new file mode 100644 index 0000000..a3963b4 --- /dev/null +++ b/neural_net/trainer.py @@ -0,0 +1,65 @@ +from neural_net.epoch import Epoch +from neural_net.neural_net import NeuralNet, ModelData + + +class NeuralNetTrainer: + def __init__(self, neural_net: NeuralNet, model_data: ModelData, learning_rate: float, batch_size: int): + self.neural_net = neural_net + self.model_data = model_data + self.is_running = False + self.epoch_history = [] + self.learning_rate = learning_rate + self.batch_size = batch_size + + def set_learning_rate(self, learning_rate: float): + self.learning_rate = learning_rate + + def set_batch_size(self, batch_size: int): + self.batch_size = batch_size + + def run_epoch(self): + epoch = Epoch(len(self.epoch_history), + self.model_data.training_inputs, + self.model_data.training_labels, + self.learning_rate, + self.batch_size + ) + self._train_one_epoch(epoch) + return epoch + + def start(self, on_epoch_finish=None, on_finish=None): + self.is_running = True + while True: + # Stop function was called causing the trainer to reset + if not self.is_running: + break + + # Perform one epoch of training + # In the future, we will apply a learning-rate algorithm + epoch = self.run_epoch() + + if on_epoch_finish is not None: + on_epoch_finish(epoch) + + if on_finish is not None: + on_finish() + self.stop() + + def stop(self): + if self.is_running: + self.is_running = False + + def _train_one_epoch(self, epoch: Epoch): + epoch.start() + + for batch in epoch.batches: + batch.predictions = self.neural_net.forward(batch.inputs) + dL_dout = self.neural_net.loss_derivative(batch.predictions, batch.labels) + + layer_dl_gradients, layer_dl_biases, layer_weights, layer_biases = self.neural_net.backward(dL_dout, epoch) + epoch.layer_dl_gradients.append(layer_dl_gradients) + epoch.layer_dl_biases.append(layer_dl_biases) + + epoch.finish(self.neural_net) + epoch.loss = self.neural_net.loss(epoch.all_predictions(), epoch.all_labels()) + self.epoch_history.append(epoch) diff --git a/neural_net/transform_layer.py b/neural_net/transform_layer.py new file mode 100644 index 0000000..76d457b --- /dev/null +++ b/neural_net/transform_layer.py @@ -0,0 +1,73 @@ +from abc import abstractmethod + +import numpy as np + +class Layer: + def __init__(self, type, index, input_dim, output_dim): + self.type = type + self.index = index + self.input_dim = input_dim + self.output_dim = output_dim + + @abstractmethod + def forward(self, inputs): + raise NotImplementedError("This should be overridden by subclasses") + + @abstractmethod + def backward(self, dL_dout, learning_rate): + raise NotImplementedError("This should be overridden by subclasses") + + @abstractmethod + def reset(self): + raise NotImplementedError("This should be overridden by subclasses") + +class TransformLayer(Layer): + def __init__(self, index, size): + super().__init__('TransformLayer', index, size, size) + + def describe(self): + return self.type + + def forward(self, inputs): + raise NotImplementedError("This should be overridden by subclasses") + + def backward(self, dL_dout, learning_rate): + return dL_dout, None, None, None, None # This is the gradient to propagate to the previous layer + + def reset(self): + pass + +class NormalizeLayer(TransformLayer): + def __init__(self, index, size): + super().__init__(index, size) + self.type = 'NormalizeLayer' + + def forward(self, inputs): + """ + Normalizes the input vector. + [1, 5, 5, 3, 6] => [0.05, 0.25, 0.25, 0.15, 0.3] + :param inputs: np.array(float) + :return: np.array(float) + """ + return inputs / inputs.sum() + +class SoftMaxLayer(TransformLayer): + def __init__(self, index, size): + super().__init__(index, size) + self.type = 'SoftMaxLayer' + + def forward(self, inputs): + """ + Normalizes the input vector, but "pushes" higher values to dominate the + probability distribution + [1, 5, 5, 3, 6] => [0.02, 0.26, 0.26, 0.10, 0.36] + :param inputs: np.array(float) + :return: np.array(float) + """ + input_ex = np.exp(inputs - inputs.max()) # Subtract max for numerical stability + s = np.sum(input_ex, axis=-1, keepdims=True) + + # To prevent division by zero, ensure that the sum is not zero + if np.any(s == 0): + return np.ones_like(input_ex) / input_ex.shape[-1] # Return a uniform distribution if sum is 0 + return input_ex / s diff --git a/test.py b/test.py new file mode 100644 index 0000000..317bcf3 --- /dev/null +++ b/test.py @@ -0,0 +1,59 @@ +import numpy as np + +# Your softmax outputs +outputs = np.array([ + [ + [ + 0.90924643, 0.0, 0.26800049, 0.0, 0.14153697, 0.07644807, + 0.0, 0.63928418, 0.14899383, 0.29679539, 0.29560591, 0.46324955, + 0.38955634, 0.0, 0.05094845, 0.0, 0.0, 0.26734416, 0.0, + 0.28399383, 0.0429699, 0.68988006, 0.0, 0.0, 0.0, 0.02901288, + 0.0, 0.01076904, 0.0, 0.41230365, 0.58630857, 0.0, 0.29906131, + 0.0, 0.00339327, 0.47909497, 0.07787446, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.59843748, 0.18691183, 0.0, 0.0, 0.0, 0.84100045, + 0.24468988, 0.0144432, 0.0, 0.27832373, 0.0, 0.45574082, + 0.16037272, 0.0, 0.28562163, 0.0, 0.0, 0.44667622, 0.0, 0.0, + 0.29725156, 0.0, 0.01500714, 0.51253602, 0.18559459, 0.07919077, + 0.0, 0.15155614, 0.0, 0.16996095, 0.26832836, 0.0, 0.56057083, + 0.47535547, 0.0, 0.08280879, 0.0, 0.07266015, 0.43079376, + 0.55633086, 0.0, 0.13123258, 0.33282808, 0.0, 0.73207594, 0.0, + 0.08246748, 0.0, 0.0, 0.0, 0.03605279, 0.56645505, 0.0, + 0.66074054, 0.0, 0.0, 0.07871833, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.26077944, 0.0, 0.0, 0.19883228, 0.26075606, + 0.0, 0.55120887, 0.0, 0.0, 0.13896239, 0.8079261, 0.0 + ], + [ + 1.3890246, 0.0, 0.0176582, 0.41937874, 0.01668789, 0.08115837, + 0.0, 0.0, 0.0, 0.03283852, 0.0, 0.28331658, 0.0, 0.56971081, + 1.29951652, 0.0, 0.05585489, 0.0, 0.0, 0.0, 0.4555721, 0.0, + 0.0, 0.0, 1.13440652, 0.3462467, 0.53066361, 0.85311426, + 0.13320967, 0.61478612, 0.0, 0.0, 0.0, 0.0, 0.0, 0.04859889, + 0.0, 0.0884254, 0.0, 0.56573542, 0.18211658, 0.0, 0.24407104, + 0.0, 0.07133323, 0.0, 0.0, 0.98712028, 0.0, 0.06996351, + 0.70575429, 0.30689567, 0.47709064, 0.07469221, 0.40548246, + 0.09671662, 0.56150121, 0.0, 0.7116001, 0.57194077, 0.0, + 0.10528511, 0.20317026, 0.03516737, 0.0, 0.0, 0.10198436, + 0.0, 0.0, 0.0, 0.35702522, 0.0, 0.0, 0.32883485, 0.0, + 0.0, 0.18996724, 0.0, 0.0, 0.0, 0.06601356, 0.0, + 0.41925782, 0.0, 0.0, 0.07929863, 0.28089351, 0.0, + 0.25405591, 0.09954264, 1.05735563, 0.0, 0.57732162, 0.0, + 0.05791431, 0.0, 0.42524903, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.13586283, 0.23484103, + 0.69677156, 0.0, 0.0, 0.08609836, 0.89882583 + ] + ] +]) + +labels = [7, 2] + +# Convert labels to one-hot encoding +num_classes = 10 +labels_one_hot = np.zeros((len(labels), num_classes)) +for i, label in enumerate(labels): + labels_one_hot[i, label] = 1 + +# Calculate the loss derivative +loss_derivative = outputs - labels_one_hot + +print("Loss Derivative:") +print(loss_derivative) diff --git a/tests/__pycache__/mnist.cpython-310.pyc b/tests/__pycache__/mnist.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..05e12be07f76ff066909cbc4536b2bf5a5eb4a71 GIT binary patch literal 2013 zcma)7O^6&t6z>1-nVns;KUsGZO;!+-5!}TUy%?Q@L_~;#uzRtIHtkeRr-z>INmWm9 zmmZMpLO_r`3F>L%L0DZ6!Gm~^Xo8>!xoCux7a?F6PYQa8NPMq)Gc!9Tl7jlCUscz8 z)$jY>n`CsfsK7X%{a$~!q$oGg7#s!|M`7|s0HP2zQr!Gkbv3ZHNN*ahp(-ybL?_0& zLJX=j%r)J$h`OYd%`Ti%s%2G-yOCM(7-qflRpYT6;~w+L{N#g zuDE*c+ck(uEU-;tgKtNSFNHxf3CT%+g|cPJQ(jZy^qpbB1J2_x`6~davZi#@RNd49 z_>>P-Kn+m4pmsD|#W2#+iPg~9e zhDkgZ^0_dsC5+KpTGoWsiu@YoLT~b*Y%+8rw74ZqpD};MH6l1H#(mByOOHj(gr_f@ zY5P&x5(cMHU6{={426wsDVN5BQ&17N8|%~Gw~ zV}_xZ^l5!{|9?oF@143godjInKbZ9;31*htuv zF%RjsB;9NjBaQ)x2hf2>dOn4XDFa5tY_0%Ls>1W75ImYIfnomHIKl+NB!D~F1MhljWcr$#8FegkGd0mVp}bZ+rFE>1O)TKt%Ivy!L^<;al)E{jbQ~ys zYp3)JpbWd=WQEMZ(%+?waxwVr^KUoLzjv{6^Wy9;=b~3Xs;~oazu@%WZ(aNT>c_by zm--M)T{*b%_>bH9X1O@9^L6u&*#l31S_uvs^Vcsww_OQp$IiXJ@#iPQj)J$J{qm=) z-+WdH+Fx~^xn^%yx;=q*F6 z9pQ7y^RS5W9X5@*L9Mex2-qJ70Lz2v0N5!w_6$dHG-skTC0EO~aN2R0;ui@MGT2&? urR|Qi(=g)mvU}zY-dTosj{DHl(cM4@8Bvy@b0ojx^Z1X_3s%ymjeh}#_Zz+d literal 0 HcmV?d00001 diff --git a/tests/__pycache__/relu_layer.cpython-310.pyc b/tests/__pycache__/relu_layer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3246a5dc7503cd89baa6240743b9517a53235c60 GIT binary patch literal 2389 zcma)-&ube;6vt78{36wAIJ>R&tNXQ@P94{1{Rd|vcAdE0ACVuwTd>wo}HhR>j8o5K5!RQmhXhgc^ zSB7sf?K9FepFtj3YihD^FN#NZ!>x$7ul<_(s;PPnikb zS<;(!vmPaB($XMhQs%(0;kgY@l0r*Lb~UE8;kT!aHRb8Krz_7;Eimyh1z#7M(D!s< zEReAYzQIgk?&)Jo#ZLOM!K1=t=0gf|H1>?K19^6x2&YcQ1!y&ejW&!~%q*cpOG)U0 zib9<%lCdicu#3trrYW3ZyAJSD6otf%B#h|$P2rYcH#j4P@TQx>I{V!W;^+&l`#?7Aur^gn%h-Lfd> zaWJE9S;c{7ag4woRo4!(4SBwj8^2EsLJt1f|NGZZ?ti+%apRt$?Ca8g9Cg|c(nMPK zJ7E$f2RncLu>0cY?{DpEl7{IZKD_+VfIkj7YYax|aFjMW{nmi-O37jAZXqnY%n@*o^W(%#6HCY}3X>me>u z7Q-Zoc)HFYASJny%q$7mcFj8nuF_(W4qf(WAOv&DA;D zzI>80OQ#HSHhI3Al85PS9r@LU_mn^;v!K>yPF|alsKAOU#$cs0SmYjO5l{^Y397Ot zOmY>I%E%<(5b&%!VUdnx>RK6(n9IB!eGCkN7{w|e50dU|M%fVIFjq0kVU?Uw)tpg< zDWi~h6GoL_L{~9tj@5us^NLaR6O77YkWpF8ImM{@2}Wfx$R0P3I-j$rE{-#566XP< zvfdUT&I3l(#Ysjju5rBC+VEgm%?)Q_R9CqP_3dxI#`4Qb`z=*|TWN2o^5;t4^yAd{ ziQAB&u*NO0;L4+L@)Pc0bpgdiDWg!@g;dIr_u)EJpp zGi$VF)Qn>%$2xX$s5&{I|KsGqfNM=tIziA6dr=U`Vi5EOY!svI2EiA=@~nlwfpc%7 z%vPZ+TnaZp@T(^Hu^r;*?j2Rvw52oZcTzmE((I3V!!7kq{i~z26DLdk{!-Y2>j18V pK|kY2vNYpIhGu6SvTwsxq`Ew9eCfQKP5Tz!VF_-snuLCy{tE#nYMuZ9 literal 0 HcmV?d00001 diff --git a/tests/__pycache__/sigmoid_layer.cpython-310.pyc b/tests/__pycache__/sigmoid_layer.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ac906091e0b53f67223466c5ee254c8f27b73056 GIT binary patch literal 2388 zcmbtVO>7%Q6rS0C+iS;hoS)KCSrDXJf z(fCBKxp9FCDLLkXxNzXa4T&q<=7fYq)dL7lNKvKm-tO8?sB>bh{r0^#-+ME&Hv7j+#|=G^RsE zL(X2TwC%@9+@g>*YO`RN@Z5zb-h-Zm98*Ty@H?SHntDp^DXFKXJuv;D2EHPwP)-z~ zE|Q_n;0G~P=p}MO7s+)pG}4?=JyXC!g|S5HWN1OAStr5*y8tXrm}tYO#mo{4^pu1m zG*PINB{H;y3U)EI%ZVjJmN6|8&%zUnr+En8@ci8MmviT^7Q%X-#>q}^YO+r+&YeMsz~#ZAG9APu{3r9 zFZSc7M}K^G{Lhcy-Z`aG^O9b8_R0r6{>bC3(HkWFLDC4KR*!SPm8>`3X(a{G&Dg%zExul9Mb z0~;2vj#jW5biKC!2JCFF9R-IzUrq3!uf~~(xM|Im{()S;xO*w=-rm1mg1Yfr02Qr5 zZK~0#R@W<L%B*|En*t{j|}3Gi|?Yw0G0?`$pdN{n+?{8<4?G1XMP-{WNbsh`EIk z{A+R>p@J}jP({Fti`Nk50GbtEM|&P&0Rir3BI!0?LI*{JUqg5a;W`4|o&05lv0Ct( z=;A5MU>U5bXJ&1GZ5#UH_W+8TL5=gGida$8v!iC7t<}^Tte6!E z$m{y9D~qn%?Xf|Kw(Yu~fuv_W{54#}EriK(QVZ9_RRCOdjGx{jmSW#aV@*?9gD6Pw z>`FZvbo&SCcXj3mNf5>>QM3~I1Kx`=Ny)_7+6MMrxPj6V)5eq5+u1bl;0DLIvFV`7 Gg7z;flV+p< literal 0 HcmV?d00001 diff --git a/tests/mnist.py b/tests/mnist.py new file mode 100644 index 0000000..b521203 --- /dev/null +++ b/tests/mnist.py @@ -0,0 +1,78 @@ +import unittest + +import numpy as np + +from neural_net.mnist import MNISTNeuralNet +from neural_net.functions.loss import cross_entropy_loss + + +# noinspection PyMethodMayBeStatic +class MNISTNeuralNetTests(unittest.TestCase): + + def test_loss(self): + mnist = MNISTNeuralNet() + # Sample predictions and labels for testing the loss function + predictions = np.array([[0.1, 0.2, 0.7], # Example of a softmax output (probabilities) + [0.2, 0.6, 0.2]]) + + # Corresponding labels (correct class indices) + labels = np.array([2, 1]) # Labels are class indices (not one-hot) + + # Expected loss (you may need to compute this manually to verify correctness) + expected_loss = cross_entropy_loss(predictions, labels) # Replace with the actual expected loss value + + # Call the loss function + computed_loss = mnist.loss(predictions, labels) + + # Assert that the computed loss matches the expected loss + self.assertAlmostEqual(computed_loss, expected_loss, places=5, msg="Loss function is incorrect") + + def test_derivative_loss(self): + mnist = MNISTNeuralNet() + # Sample predictions and labels for testing the derivative of the loss function + predictions = np.array([[0.1, 0.2, 0.7], # Example of softmax output (probabilities) + [0.2, 0.6, 0.2]]) + + # Corresponding labels (correct class indices) + labels = np.array([2, 1]) # Labels are class indices + + # Expected derivative of loss (manually computed or from a trusted source) + expected_derivative = np.array([[0.1, 0.2, -0.3], # Replace with actual expected gradient + [0.2, -0.4, 0.2]]) + + # Call the derivative loss function + computed_derivative = mnist.loss_derivative(predictions, labels) + + # Assert that the computed derivative matches the expected derivative + np.testing.assert_array_almost_equal(computed_derivative, expected_derivative, decimal=5, + err_msg="Derivative of loss function is incorrect") + + def test_derivative_loss2(self): + mnist = MNISTNeuralNet() + + # Given outputs + outputs = np.array([ + [0.06873367, 0.043651, 0.043651, 0.05235898, 0.043651, 0.043651, + 0.043651, 0.043651, 0.0563062, 0.043651], + [0.043651, 0.043651, 0.05704588, 0.0551587, 0.05460022, 0.043651, + 0.043651, 0.043651, 0.07723706, 0.05474726] + ]) + + # Labels + labels = [7, 2] + num_classes = 10 + + # Convert labels to one-hot encoding + labels_one_hot = np.zeros((len(labels), num_classes)) + for i, label in enumerate(labels): + labels_one_hot[i, label] = 1 + + # Calculate the expected loss derivative + expected_loss_derivative = outputs - labels_one_hot + + # Call the derivative loss function + computed_loss_derivative = mnist.loss_derivative(outputs, labels) + + # Assert that the computed derivative matches the expected derivative + np.testing.assert_array_almost_equal(computed_loss_derivative, expected_loss_derivative, decimal=5, + err_msg="Derivative of loss function is incorrect") diff --git a/tests/relu_layer.py b/tests/relu_layer.py new file mode 100644 index 0000000..76624ee --- /dev/null +++ b/tests/relu_layer.py @@ -0,0 +1,154 @@ +import unittest + +import numpy as np + +from neural_net.activation_layers.relu_layer import ReluLayer + + +# noinspection PyMethodMayBeStatic +class ReluLayerTests(unittest.TestCase): + + def test_relu_layer_1x1(self): + ############## + # Arrange # + ############## + inputs = np.array([[1.0]]) + weights = np.array([[0.5]]) + biases = np.array([0.0]) + learning_rate = 0.001 + + # Pre-activation value (z) + # This is the intermediate value calculated as the weighted sum of inputs plus the bias. + z = np.dot(inputs, weights) + biases + + # ReLU activation: f(z) = max(0, z) + # The expected output after applying the ReLU activation function + expected_output = np.maximum(0, z) + + # Loss gradient dL/dout + # Represents how much the loss changes when the output changes. + dL_dout = np.array([[1.0]]) + + # Activation derivative dout/dz + # For ReLU: If z > 0, dout/dz = 1; otherwise, dout/dz = 0 + dout_dz = np.where(z > 0, 1.0, 0.0) + + # Gradient of the loss with respect to weights (dL/dweights) + # This represents how much the loss changes when the weights change. + # Formula: dL/dweights = inputs × dL/dout × σ′(z) + expected_dl_dweights = inputs * dL_dout * dout_dz + # Gradient of the loss with respect to the bias (dL/dbias) + expected_dL_dbias = np.sum(dL_dout * dout_dz) + + # Gradient of the loss with respect to inputs (dL/dinputs) + # This is the gradient of the loss with respect to the input of the neuron or layer, often needed if you want to backpropagate further. + # Formula: dL / dinputs = dL/dout × σ′(z) × weights + expected_dl_dinputs = dL_dout * dout_dz * weights + + # Calculate expected new weights and biases + expected_weights = weights - learning_rate * expected_dl_dweights + expected_biases = biases - learning_rate * expected_dL_dbias + + # Initialize SigmoidLayer + layer = ReluLayer(weights.shape[0], weights.shape[1], weights=weights, biases=biases) + + ############## + # Act # + ############## + # Forward pass + output = layer.forward(inputs) + + # Backward pass + dl_dinputs = layer.backward(dL_dout, learning_rate) + + ############## + # Assert # + ############## + ############## + # Assert # + ############## + # Forward output correctness + self.assertTrue(np.allclose(output, expected_output, atol=1e-6), + f"Forward output incorrect: Actual: {output}, Expected: {expected_output}") + + # Backward pass correctness + self.assertTrue(np.allclose(dl_dinputs, expected_dl_dinputs, atol=1e-6), + f"Inputs derivative incorrect Actual: {dl_dinputs}, expected: {expected_dl_dinputs}") + self.assertTrue(np.allclose(layer.weights, expected_weights, atol=1e-6), + f"Weight update incorrect Actual: {layer.weights}, expected: {expected_weights}") + self.assertTrue(np.allclose(layer.biases, expected_biases, atol=1e-6), + f"Bias update incorrect Actual: {layer.biases}, expected: {expected_biases}") + + def test_relu_layer_2x2(self): + ############## + # Arrange # + ############## + inputs = np.array([[1.0, 2.0], + [3.0, 4.0]]) # 2x2 input matrix + + weights = np.array([[0.5, 0.2], + [0.3, 0.7]]) # 2x2 weight matrix + + biases = np.array([0.1, -0.1]) # 2 biases, one for each neuron + + learning_rate = 0.001 # Learning rate for weight updates + + # Pre-activation value (z) + # z = inputs.dot(weights) + biases + z = np.dot(inputs, weights) + biases + + # Expected output using the ReLU activation function + expected_output = np.maximum(0, z) # Apply ReLU + + # Loss gradient dL/dout (assuming a gradient of 1 for simplicity) + dL_dout = np.array([[1.0, 1.0], + [1.0, 1.0]]) + + # Activation derivative dout/dz + # For ReLU: dout/dz = 1 where z > 0, and dout/dz = 0 where z <= 0 + dout_dz = np.where(z > 0, 1.0, 0.0) + + # Expected gradients (for backpropagation) + # Expected gradients with respect to weights + expected_dl_dweights = np.dot(inputs.T, dL_dout * dout_dz) + + # Expected gradients with respect to biases + expected_dL_dbias = np.sum(dL_dout * dout_dz, axis=0) + + # Expected gradients with respect to inputs + expected_dl_dinputs = np.dot(dL_dout * dout_dz, weights.T) + + # Expected updated weights and biases after backpropagation + expected_weights = weights - learning_rate * expected_dl_dweights + expected_biases = biases - learning_rate * expected_dL_dbias + + # Initialize the ReLU Layer + layer = ReluLayer(weights.shape[0], weights.shape[1], weights=weights, biases=biases) + + ############## + # Act # + ############## + # Forward pass + output = layer.forward(inputs) + + # Backward pass + dl_dinputs = layer.backward(dL_dout, learning_rate) + + ############## + # Assert # + ############## + # Forward output correctness + self.assertTrue(np.allclose(output, expected_output, atol=1e-6), + f"Forward output incorrect: Actual: {output}, Expected: {expected_output}") + + # Backward pass correctness (for input gradients) + self.assertTrue(np.allclose(dl_dinputs, expected_dl_dinputs, atol=1e-6), + f"Inputs derivative incorrect Actual: {dl_dinputs}, Expected: {expected_dl_dinputs}") + + # Check weight updates + self.assertTrue(np.allclose(layer.weights, expected_weights, atol=1e-6), + f"Weight update incorrect Actual: {layer.weights}, Expected: {expected_weights}") + + # Check bias updates + self.assertTrue(np.allclose(layer.biases, expected_biases, atol=1e-6), + f"Bias update incorrect Actual: {layer.biases}, Expected: {expected_biases}") diff --git a/tests/sigmoid_layer.py b/tests/sigmoid_layer.py new file mode 100644 index 0000000..02fc023 --- /dev/null +++ b/tests/sigmoid_layer.py @@ -0,0 +1,142 @@ +import unittest + +import numpy as np + +from neural_net.activation_layers.sigmoid_layer import SigmoidLayer + + +# noinspection PyMethodMayBeStatic +class SigmoidLayerTests(unittest.TestCase): + + def test_sigmoid_layer_1x1(self): + ############## + # Arrange # + ############## + inputs = np.array([[1.0]]) + weights = np.array([[0.5]]) + biases = np.array([0.0]) + learning_rate = 0.001 + + # Pre-activation value (z) + # This is the intermediate value calculated as the weighted sum of inputs plus the bias. + z = np.dot(inputs, weights) + biases + + # Ouput + # The result of applying the activation function to the pre-activation value z + # Sigmoid activation formula: 1 / (1 + e^-z) + expected_output = 1 / (1 + np.exp(-z)) + + # Loss gradient dL/dout + # Represents how much the loss changes when the output changes. + dL_dout = np.array([[1.0]]) + + # Activation derivative dout/dz + # This tells you how much the output of the activation function changes with respect to the pre-activation value z. + # Sigmoid derivative formula: σ(z) * (1 - σ(z)) + dout_dz = expected_output * (1.0 - expected_output) + + # Gradient of the loss with respect to weights (dL/dweights) + # This represents how much the loss changes when the weights change. + # Formula: dL/dweights = inputs × dL/dout × σ′(z) + expected_dl_dweights = inputs * dL_dout * dout_dz + # Gradient of the loss with respect to the bias (dL/dbias) + expected_dL_dbias = np.sum(dL_dout * dout_dz) + + # Gradient of the loss with respect to inputs (dL/dinputs) + # This is the gradient of the loss with respect to the input of the neuron or layer, often needed if you want to backpropagate further. + # Formula: dL / dinputs = dL/dout × σ′(z) × weights + expected_dl_dinputs = dL_dout * dout_dz * weights + + # Calculate expected new weights and biases + expected_weights = weights - learning_rate * expected_dl_dweights + expected_biases = biases - learning_rate * expected_dL_dbias + + # Initialize SigmoidLayer + layer = SigmoidLayer(weights.shape[0], weights.shape[1], weights=weights, biases=biases) + + ############## + # Act # + ############## + # Forward pass + output = layer.forward(inputs) + + # Backward pass + dl_dinputs = layer.backward(dL_dout, learning_rate) + + ############## + # Assert # + ############## + # Forward output correctness + self.assertTrue(np.allclose(output, expected_output, atol=1e-6), + f"Forward output incorrect: Actual: {output}, Expected: {expected_output}") + + # Backward pass correctness + self.assertTrue(np.allclose(dl_dinputs, expected_dl_dinputs, atol=1e-6), + f"Inputs derivative incorrect Actual: {dl_dinputs}, expected: {expected_dl_dinputs}") + self.assertTrue(np.allclose(layer.weights, expected_weights, atol=1e-6), + f"Weight update incorrect Actual: {layer.weights}, expected: {expected_weights}") + self.assertTrue(np.allclose(layer.biases, expected_biases, atol=1e-6), + f"Bias update incorrect Actual: {layer.biases}, expected: {expected_biases}") + + def test_sigmoid_layer_2x2(self): + ############## + # Arrange # + ############## + inputs = np.array([[1.0, 2.0], + [3.0, 4.0]]) + + weights = np.array([[0.5, 0.2], + [0.3, 0.7]]) + + biases = np.array([0.1, -0.1]) + learning_rate = 0.001 + + # Pre-activation value (z) + # z = inputs.dot(weights) + biases + z = np.dot(inputs, weights) + biases + + # Expected output using the sigmoid function + expected_output = 1 / (1 + np.exp(-z)) + + # Loss gradient dL/dout (assuming a gradient of 1 for simplicity) + dL_dout = np.array([[1.0, 1.0], + [1.0, 1.0]]) + + # Activation derivative dout/dz + dout_dz = expected_output * (1 - expected_output) + + # Expected gradients + expected_dl_dweights = np.dot(inputs.T, dL_dout * dout_dz) + expected_dL_dbias = np.sum(dL_dout * dout_dz, axis=0) + expected_dl_dinputs = np.dot(dL_dout * dout_dz, weights.T) + + # Expected updated weights and biases + expected_weights = weights - learning_rate * expected_dl_dweights + expected_biases = biases - learning_rate * expected_dL_dbias + + # Initialize SigmoidLayer (assuming SigmoidLayer class exists) + layer = SigmoidLayer(weights.shape[0], weights.shape[1], weights=weights, biases=biases) + + ############## + # Act # + ############## + # Forward pass + output = layer.forward(inputs) + + # Backward pass + dl_dinputs = layer.backward(dL_dout, learning_rate) + + ############## + # Assert # + ############## + # Forward output correctness + self.assertTrue(np.allclose(output, expected_output, atol=1e-6), + f"Forward output incorrect: Actual: {output}, Expected: {expected_output}") + + # Backward pass correctness + self.assertTrue(np.allclose(dl_dinputs, expected_dl_dinputs, atol=1e-6), + f"Inputs derivative incorrect Actual: {dl_dinputs}, expected: {expected_dl_dinputs}") + self.assertTrue(np.allclose(layer.weights, expected_weights, atol=1e-6), + f"Weight update incorrect Actual: {layer.weights}, expected: {expected_weights}") + self.assertTrue(np.allclose(layer.biases, expected_biases, atol=1e-6), + f"Bias update incorrect Actual: {layer.biases}, expected: {expected_biases}") diff --git a/ui/__pycache__/app.cpython-310.pyc b/ui/__pycache__/app.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5517ad5455c7ccb565a164b29b6c37e97715a1cf GIT binary patch literal 853 zcmYjQyKWRQ6tz7wyWYtr2ow+nMcOHn4G|IzLOdi81rbuVa$^l+?=rid2ewyAR_PKc zRsJC<^FLHBfhNB|#l5qEtR>$$zT-JQ=UCHhZl(mr@1MQt4@$@{T&#~67H?p>cTf~j z)DS`8$qcI^5mD?#Caw~ZbT~G>sxSJKd?YGSy>p^^n$GxHBnFBek*t3UJmfH=fxm6r zBcFQ>c<{kCwf~Y&agmfoQ@aB9t&iw#;hq7|oeIG&$b>4Y*f|%`1&N7>;fyFmz+Hfy z{{t~vK@vBH;>j7gqSu5H&ye7fo)Q`P_$tBPDY@#Y=$dw@uVO@=pF)2YoiW%s05q7T zXi@bq8Cr*f`QbRDVvsLSwXpN_^mW1E+CSx)LB8`1AdP7soFT_HR*F zh`%=eTgF09AGdj}LOdyr$u>gb7Ol2{O9@|lDJ_HDK;O2~VQ>~5i4;$gU1lNir8ipe zsctIm?TJOdBAsBmvW0j<+~&m`LXMp_lN(50%EIK%NqIy5?2cO7%yi-1$mran6>;KjB$|ChEM>QifE@ zs!Qg=VY69oh%J~KQA%h!2Dbc z7v?0d8eHYX;dPheDp)q2cf@NNIyI;v@K-Anuki*Q{zIG$*$_f_68!~* CkS!kO@I3s@B<1F2jNm-|{_FUmkI)aNbdLwhOCa+am_Pz6 zL@|65MRJdN*x5Yg=RMkMWnTq(NW(m$5k~Kj=!xJQiQtUTzQ9wIgg+qj)2|(nXkZRXo(nFV*SG6Gba;)zYyzLf8M(YmPC+CvX;G$?04keB{}mU zHe8}Pxrl&=^M1?YOM)RQSU3K)3N!^c+lr2dPEUf#;__D7x%5^SuxCVK9gpciO>7P`smckYlq@TQnik~E61+gd-Ca)fR+8f`^kVtZ4If7 zc9_w)Xa~g#X|)5iA*E^jirbHkKQFU_#<`mkIddH_xQ-oFK_7L_4nEN4Zw zka}EaV_sFvxNR6!r%lA5iOe$Q7Qs9M7GNJoKm-r2KEa>e-~kRuj2AoCn>4zPgMB_o ze3z%-xn!(~8H3l?%E<%9PHL`J9=!$G+g3Yl)7b)GA&`NeA#CIpZY@dNgryldShe%g z9rx%_USxorLqX~meY$-!j+J*crF$Loji_A>-E+*9?X==R)UKZKhAG=z~vb0fzVBi>mX+*n`5Ia6NhV?)cA=W~%4Fn(%3}&Z$$)Scb zbWc-D65Nvn_>%n(>vP*XeHXn`cw@iskAr@|Y2Gcuanz4Ed&-0-{5MSal8;)i z-F{o}?^rK*hjZ9b&oQlM<8&aOs`S`|cKH3v9yjf8Wu1;EMP3dvTDkqX8>W7OfwX$` zogdi=7hE`RBIx4^?+xqYUJ;=8MJOWlfoO>~`cQO47k%_2^H{$HZFWrONj}K+6P2Dy zm65R*AC2ikYW1fen!RQ#zT|7p+0t3E#96wauHc~FlCAtTBM5M;bM%h3^#bG9(}`3j zN|L0C(}_s6Os09yGhQu=6BA5Q zB}=W~C#J}4Z@FnDNmitFog{DBe;*!BR5g;Bt`7^DsuCa`=7jP+C9`T!;#_q&%@4C` zJgG_?RUZm6KM~gcgUMV)IF?#X=XZHve_g(~xrNigg*6$+qKnVLfUJIuR^I^uBy7!| zu)haqfNjMIo*z655T^jxJL4V7m!`(Lic-9ZW-RkT}Z7 zHsEp%TpfUPd7FFxdtb$iMenId>e2?Q;y*BGROC$a5j?tNBY1S=Q8=B$qlpjB9|Z_2 z1lGB8u@2$oU#On;vpr6xGG~klyPHed_0dcB~y(D7FVy*qK-ewas@EzUYzQ9Tuo~kPpUd6m=BD9 zTmVAE^{=6#39G3_ev?N%-|(60kc`jOZ>aCsV`V~z_JZae`0^E=s&QubjWb7^JwX59 zDwe;+_K(o)i@4+5;dlAs`Zj{r5kGqlF1A?o$OS+GDNfiFCmdMsF8*MnYuk2R#c5Gg zr&b0#RSXFwkT{J?c^cb{a$sC6!Ygmg3WO}Amg*iZxvKc5kbQzylVkUs#s0R^+wt|D z?&S5$r~FSa%~K|iubefz&5(20%3a%NSb0|BEFr@@5;f*sO5)%G%XvCnMVPlyaN@1` zszo@A+QQMDQCGO<&Y7=wR=Z1oO$q-R_U?_YqsDpq#^?iZ*MxuWAw}F=eMoCW@Jn_U z=v&}#F9V#nFPt@hO~oTv{bJ3|!et1A!^P*1tMZjpdK+VclVXdojvst6d+Tl`SON)gYV+^ zvD@YQeAl_jBgf;5yYB^VgT&?LUYq^|l<8RFPe40K#+8^BH0~zJ57V@0W^DPjIoak$ zMOP|%AnLCmW@l6EGg%Z#vZXMULPgb4`5+-$y$cedG)7_MMc1NW&$DyAjtTWk04SBR z`1<8^jJp39EI00pXWv+VT37X3U=OUP8b|tnQqbQXvvs#E&rN0iva#wbia+}YB4QD8 JdY9jF{tq#DsKfvO literal 0 HcmV?d00001 diff --git a/ui/components/__pycache__/label_with_refresh.cpython-310.pyc b/ui/components/__pycache__/label_with_refresh.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53013cb6138514bd6c4cb89d890e8e4d31032593 GIT binary patch literal 1336 zcmZ`(OHUgy5VpOKY|?~MRMA4EKImyHmE5bUYAG#6LODRCN_(-EwG%dMc2lpT(r9lW zams&yBY(+PPMo>*)ES2$9<8Mvk3Hjgd^58}tJNT|TIYU`I)t3#;pTGT@DaZ94G=*D z%}7LHXBitfkweJ_5sq+=h;SwCyN6EX3A#ffQ*os8YDM}b8&$=gbW}} zg*5Ik76$;Ai%>$Lncg@4+DPjnR}2)V=+0dk<-1mbub@QU_U6X7@5ZzF8_yyZIx4_t z*zxKClDTX05V;xpEIGo7%36O+dS2@})cd{Br{hJ2IrpQAXNO4L)Z zVuJO}olk39U)F8wTJ+T`^zj;{t*aqPWR`J$Bd8{-*m@z@*fAwmGyqTnFYxCbh4-Ov yYI;A-wN%zyG&f3D?XPG`Z3h|etAbGNgBuMaC^k+2 literal 0 HcmV?d00001 diff --git a/ui/components/__pycache__/number_slider.cpython-310.pyc b/ui/components/__pycache__/number_slider.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1fe4f48e65d8244c31efddfadbd353c02de6cdd2 GIT binary patch literal 1069 zcmY*Yxo+Gr5G8f3_DTW+*&;=%Ab^T^e;^2QAU=R#4cM;K5VFW!Yt5AulJW(0HsH#? z5Ufl8Nxp%tEB`{O%xKq!Okt>*;c%`uEQZ4#fi-#vW`7()9#Cj64}`Bk`aNu%a4Jbm zw+VN+dri12=-jz-W0%td5_ykcLnbNi?d7fo{Q(#O_c@838#1Mwa_2gP+b;L6N$lMa zkHkKB^6g+!EslgbkQo;$4O;Gy_kqxBDH4@u`D~mml9~9VL|V-9>{6(4m5tM4Q5HEk z*Wyph5Qic2ar~<88W9esma{o^cH*K z0`JZ}(EEtptwYf!02^96+^b#ggWTZ(e4XZAc-x12A@Jd{i<)Kcjx;?6%}Y;Bhr5%A z+JO=0<}^`RawNqgHZO9ZqX44y%7mXk{sq;R^`XLQ3T+@oJ~PLv4?^74 z$HiHLPaG5~gMP(>Ril)UnE;5^|Nc>-U`Wm~ZjNpE6g=t!UMJT72_RmqVMIVLFp(4v z_II~;4y}KXCQ{hm&fflRyf-=g_A3rFs+4uLFxH(36M3qGIX)-aB&c4YuGwBAhlrtt z#!@x_xQ{?1dO3OlUwhJE4)bIoU=DkXEec*q3xJ)*X z$y;qX)#Z+{YmywE9 zazn~kah1ZFd>}C?Ie5=~j%Lq|Tlr^Md=t4!t%$6Qb!F zCxk!hWwjE$+4QoOPW#i!XjnxLfr>?_k>?Lj^F}?cgF{}qus8O)_ zyhstc9lRfa?nsFBMhz|UVdZ=fBzQ+Cflt7Kj88hCP`w|hXLyZ-6W pOz}P#w0p+j55Onk&k(#G1o@?vKKp)`?+e6Ucn}#KvoXwP>|bu!PV4{x literal 0 HcmV?d00001 diff --git a/ui/components/digit_drawer.py b/ui/components/digit_drawer.py new file mode 100644 index 0000000..a5409d5 --- /dev/null +++ b/ui/components/digit_drawer.py @@ -0,0 +1,61 @@ +import tkinter as tk + +import numpy as np +from PIL import ImageGrab, ImageTk +from PIL.Image import Resampling + +class DigitDrawer(tk.Frame): + def __init__(self, parent, canvas_width, canvas_height): + super().__init__(parent) + self.canvas_width = canvas_width + self.canvas_height = canvas_height + self.brush_size = 3 + self.update_ui() + + def clear_ui(self): + for widget in self.winfo_children(): + widget.destroy() + + def update_ui(self): + self.clear_ui() + # Create a Canvas to draw on + self.canvas = tk.Canvas(self, width=self.canvas_width, height=self.canvas_height, bg='white') + self.canvas.pack(padx=10, pady=10) + self.canvas_demo = tk.Canvas(self, width=28, height=28, bg='white') + self.canvas_demo.pack(padx=10, pady=10) + + # Clear Button + self.clear_button = tk.Button(self, text="Clear", command=self.clear_canvas) + self.clear_button.pack(expand=True, fill='both') + + # Bind mouse events to draw on the canvas + self.canvas.bind("", self.paint) + + def paint(self, event): + """Draw on the canvas by creating ovals (circles) at mouse position.""" + x1, y1 = (event.x - self.brush_size), (event.y - self.brush_size) + x2, y2 = (event.x + self.brush_size), (event.y + self.brush_size) + self.canvas.create_oval(x1, y1, x2, y2, fill='black', outline='black') + + def clear_canvas(self): + """Clear the canvas to allow the user to draw a new digit.""" + self.canvas.delete("all") + + def convert_to_array(self): + """Convert the canvas drawing to a 28x28 grayscale array.""" + # Get the canvas's pixel data and save it temporarily + x = self.winfo_rootx() + self.canvas.winfo_x() + y = self.winfo_rooty() + self.canvas.winfo_y() + x1 = x + self.canvas.winfo_width() + y1 = y + self.canvas.winfo_height() + + # Capture the canvas area and convert it into a grayscale image using PIL + image = ImageGrab.grab((x, y, x1, y1)).convert("L").resize((28, 28), resample=Resampling.HAMMING) + self.demo_image = ImageTk.PhotoImage(image) + self.canvas_demo.create_image(0, 0, anchor=tk.NW, image=self.demo_image) + + image_array = np.asarray(image) / 255.0 + print(np.array(image_array).reshape((28, 28))) + + flat_array = image_array.flatten() + return flat_array diff --git a/ui/components/label_with_refresh.py b/ui/components/label_with_refresh.py new file mode 100644 index 0000000..78b0b4c --- /dev/null +++ b/ui/components/label_with_refresh.py @@ -0,0 +1,21 @@ +import tkinter as tk + +from ui.icons.icons import icons + +class LabelWithRefresh(tk.Frame): + def __init__(self, parent, initial_text, callback, initial_state=tk.DISABLED): + super().__init__(parent) + self.callback = callback + self._create_ui(initial_text, initial_state) + + def _create_ui(self, initial_text, initial_state): + self.refresh_button = tk.Button(self, image=icons["refresh"], state=initial_state, command=self.callback) + self.refresh_button.pack(side=tk.RIGHT, padx=5) + self.label = tk.Label(self, text=initial_text) + self.label.pack(side=tk.RIGHT, padx=5) + + def set_state(self, state): + self.refresh_button.config(state=state) + + def set_text(self, text): + self.label.config(text=text) diff --git a/ui/components/number_slider.py b/ui/components/number_slider.py new file mode 100644 index 0000000..ea1d8b2 --- /dev/null +++ b/ui/components/number_slider.py @@ -0,0 +1,14 @@ +import tkinter as tk + +class NumberSlider(tk.Frame): + def __init__(self, parent, value, from_, to, resolution): + super().__init__(parent) + self.value = value + self.update_ui(from_, to, resolution) + + def update_ui(self, from_, to, resolution): + self.entry = tk.Entry(self, textvariable=self.value) + self.entry.pack(side=tk.RIGHT, padx=5) + self.scaler = tk.Scale(self, from_=from_, to=to, length=200, resolution=resolution, showvalue=False, orient=tk.HORIZONTAL, sliderrelief="flat", relief="flat", borderwidth=0, variable=self.value) + self.scaler.set(self.value.get()) + self.scaler.pack(side=tk.RIGHT, padx=5) diff --git a/ui/components/plot_figure.py b/ui/components/plot_figure.py new file mode 100644 index 0000000..4c87a37 --- /dev/null +++ b/ui/components/plot_figure.py @@ -0,0 +1,27 @@ + +import tkinter as tk + +from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg +from matplotlib.figure import Figure + +from ui.plotters.plotter import Plotter + +class PlotFrame(tk.Frame): + def __init__(self, parent, width=None, height=None): + super().__init__(parent, width=width, height=height) + if width is not None or height is not None: + self.pack_propagate(False) + self.figure = self.create_plot_figure() + self.plotter: Plotter = None + + def create_plot_figure(self): + figure = Figure(layout="compressed", facecolor=(0,0,0)) + # Create a matplotlib canvas to display the plot + canvas = FigureCanvasTkAgg(figure, self) + canvas.draw() + (canvas.get_tk_widget() + .pack(fill=tk.BOTH, expand=False, padx=0, pady=0, ipadx=0, ipady=0)) + return figure + + def update_data(self, data): + self.plotter.update_plot(data) diff --git a/ui/front_page/__pycache__/front_page.cpython-310.pyc b/ui/front_page/__pycache__/front_page.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..379e5d26192facc303f980827037b3225ce5974a GIT binary patch literal 3062 zcmZ`*OK%%D5GGfv)k<1D>;#U32CW*jO}D7+rjJ8`zS6W#(AVp{@-v%?`docTDzQLR>C`2G1;d7F8R z{X-uoe>Oh8K}#Qj2qt*Kg1qN}g>Ne<4D7(>Br7DvffG1}x0BMq4P3()lk%VvR1EJV z)qxjyoZVrfB;2P=xYFv^_6k8=@Ga(-Utk~B_PMU#Z{OYO+#d;3UAB<`*8cq@xC z3BJ~r6BQ+GncW@sMi4YRGR=(kmh5KnXb4%OqoQ~i54Y!{F0Fm?N2jA-8h}U^aKUgY zE(<;D8S8OjiNaGau!Sv(PgzhDjwqpbge%JEOQIsG=w0E78v3%Riw63NXo@BD)o09R zfrnEq>++@=4YP;Qw$$ZlJPuPjN4H>h&k6C1mbUScu|2lWXZ(P3HsdoES~Cl@AQ-4^ zXwgu|(2}99q2-*PvHi*cBfoF4L*DkQx|mMJQfW5~;g>87wKI;C9A?VJhPpCY3_TyoFN)vK4T$kp3y3Im4q9@ zN$eN2oyw%AfI()AIpn$=hTSAe(=a?_|9-tOR--|N-ElM&fjy4IZpXLOo6u2j(d3%8vK?K#7d?`R#&~x} z7dj6fYWv#ZWss|ADIKcL>)d(e6_)v`wMtREbR5MafSn1Wg_$OdEiiC5F3E(e3UT|K zTntxR3s9<&tTP$o08+gVnevIeX59yny@Qq#R4TkeHZ_lJ%BP<(=m=MTW7gat zrXGUhePmh~m-D#)#Ai0)s_z(N8U21Kv->Vud5%XypE(e%fmYos09zh7&*;`()2(BK zR^oe^gVPR>)_HJWk#`k2OxL3xLUELU)Dj8pd*5$%6oFe2Ky>lv zyLW!*D2mIu0i#G#Ak$Tl>wwcLzYHift!unuQ2^`Qel@**9Kws>dM2m~l)b!+l`|xY zJxb$p|v-#)aPg~>6W$T zDVg)l)yI&Y>!J5B@Ffj?3NpQVYQH>Yx%sCyq{CB)%{BWGjNpIpjAg*{eqn~3d0-v! zg%iAef-fFelo9)O2(<4ziwo&H?djX+R0Kj!Dp4Z_@i5J%S4_!SGj$?ei^c9IYm-M$ zGPxK?D3zIkoAZH{?B+*1=S;}uXY~D&qEl*Xlu|~_e${q=6b}uu)rS}{VMG?CET*eu zUf3HYLaNu2hF&(9Ix3N*c?)w)hO@baFz0{wKhK$7I}X6vX{Ta96>9;)l!`07$QEAA z(;B}RYU`Nr6Inq^oCezIdxzS9H^mSI0B?#)NU4015buTIZ<8p=M-;tpOd&QIJm1%( zQNpw$kSRj8qJ4F3UjDl>Ny6}izvwP0QwK`mDrhMc1eYyyx9(QmqT6(B*KSr6jr$Gt z89H6cc5toW_!fOFk%QLpAX6kEMDX z%ikAIA5;4R|AIMZpG7831vf6zf9+vql3tz_>1-#QVrmdMMY~_*-v5eXZ}8(%ny?lK HTCx5IZ+7C$ literal 0 HcmV?d00001 diff --git a/ui/front_page/front_page.py b/ui/front_page/front_page.py new file mode 100644 index 0000000..4abbadc --- /dev/null +++ b/ui/front_page/front_page.py @@ -0,0 +1,76 @@ +import os +import tkinter as tk + +from data.mnist_loader import MNISTModelData +from ui.app_state import AppState +from ui.front_page.sections.model_overview_section import NeuralNetInfo +from ui.front_page.sections.test_model_section import TestModelSection +from ui.front_page.sections.training_section import TrainingSection + +class FrontPage(tk.Frame): + def __init__(self, parent, app_state: AppState): + super().__init__(parent) + self.parent = parent + self.app_state = app_state + + self.main_frame = None + self.neural_net_info = None + self.model_actions_frame = None + self.start_training_section = None + self.test_model_section = None + self.training_section = None + self.test_model_section = None + self.create_ui() + + def create_ui(self): + (tk.Label(self, text="Welcome to MNIST Learning Center", font=("Arial", 16)) + .pack(side=tk.TOP, fill=tk.BOTH, expand=False, padx=5)) + + self.main_frame = tk.Frame(self) + self.main_frame.pack(fill=tk.BOTH, expand=True) + + self.neural_net_info = NeuralNetInfo(self.main_frame, self.app_state, self.on_model_loaded, self.on_data_loaded) + self.neural_net_info.pack(side=tk.TOP, fill=tk.BOTH, expand=True, padx=5) + self.load_model_actions_frame() + + def update(self): + if self.neural_net_info is not None: + self.neural_net_info.update() + self.load_model_actions_frame() + + def load_model_actions_frame(self): + if self.model_actions_frame is None and self.app_state.neural_net is not None and self.app_state.model_data is not None: + self.model_actions_frame = tk.Frame(self.main_frame) + self.model_actions_frame.pack(side=tk.BOTTOM, fill=tk.BOTH, expand=True, padx=5) + + self.training_section = TrainingSection(self.model_actions_frame, self.app_state, self.after_training) + self.training_section.pack(side=tk.LEFT, fill=tk.BOTH, expand=True, padx=5) + + self.test_model_section = TestModelSection(self.model_actions_frame, self.app_state) + self.test_model_section.pack(side=tk.RIGHT, fill=tk.BOTH, expand=True, padx=5) + else: + if self.test_model_section is not None: + self.test_model_section.update() + if self.training_section is not None: + self.training_section.update() + + def on_data_loaded(self): + print("Data loaded") + self.update() + + def on_model_loaded(self): + print("Model loaded") + self.update() + + def load_training_data(self): + data_folder = "/projects/learning/datasets/minst" + self.app_state.model_data = MNISTModelData( + os.path.join(data_folder, "train-images-idx3-ubyte"), + os.path.join(data_folder, "train-labels-idx1-ubyte"), + os.path.join(data_folder, "t10k-images-idx3-ubyte"), + os.path.join(data_folder, "t10k-labels-idx1-ubyte") + ) + self.update() + + def after_training(self): + self.update() diff --git a/ui/front_page/plots/__pycache__/gradients.cpython-310.pyc b/ui/front_page/plots/__pycache__/gradients.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5d9db95f19ec40d340acdf885276a80862dbee76 GIT binary patch literal 2305 zcmaJ?OOG2x5bmCr$FIf7W=$LdS`ZQ%9=3?Yjdvg+2subx&|HS`blW?to*C0UqwHGd zv@7M58$Td>%wOs&64Hu)zy+#$)-N`pttnS^RlmOay1M9gTLju)zc*)}d4&9d#&&bT z_!2&qgJ6VFK{6Urin=AN%nog%+rr7*&^5XvysQy6jP43QYlcmud!m)K!!{)k32QL_ zoG_m^&+M=Rc9XThZt*s!XHM8QirlQIa$AoG$4Fz*SmHa%?ZHGx(1jMqpw( z4qWZ3YR;whqbSYOGKzF>XE8W{_NM08x-qX$6xiBPTuik$k0tC%Hesz7aQrArM66U4 zy&!*oGn&idm?xzg2_DNl&1a)Di)Z{J$&+H1rwcAeRXUo=A}^ykI7T?V8qHQ`9nMc> z8)n8&A!r?XlP>x@+Z(R79e8+z9H56Iw4x|0SS3*JMA5HRENYKLHknCv_iD2V7b1$} zK01uJf!Zw;J_sbIS==XgsOfoiKEH?9?R+weJ5$Ib9$pq==HtH748VZeF=(m-QCGWcL(dhjW@w~!nU(=*sf53pyQ zp#)c6HPmtI_u*5&fBp2@ah8x2jtjVTFgNfK7*=hVxZT=*d_)t05QPURm=)I#OCrJXa+^?Cq*Hi z#4-b_RAmuZ+DWo_uDw{z;m$*axGCaOE+ZzQ^@XnS^3qHl6;A*@!>+esV%=NEL}u8> z*Lv=4-B5#|gV?26XGiqEE|{rLqJ*GZZy;8h_Csohx2 zk7Di4Bd)kgCiA3(}8(Cc9c|WtgR2Wx;LuXrXE!3Fe$Qm zk>e{nG$>t7iQLfp;KxoVXtfs<*#M7u66!*|IgHT^OZ`vlhv;*xJqn9uhjc9XQT+9+ GzWXofzZ_El literal 0 HcmV?d00001 diff --git a/ui/front_page/plots/__pycache__/layer_weights.cpython-310.pyc b/ui/front_page/plots/__pycache__/layer_weights.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eae9e91238ad66ee644f59dc5751ba967d3bf0c8 GIT binary patch literal 2092 zcmZuyOK&4Z5bmDod3Ze8I0T|3EIFWMB@3;?i338wvS{IeETmltX(ZH|Ot+meGh?TF zOtR7VLL%*n3*rLekR0=u`pNu$S&0a=zkh2DfAt9Y z3nz;W*2ovo_0J##5wsvBO(;d(DV)TC(JkDPCCsj|!Ylp6x4KsZWtfCk_ls5;C6U#G zqFu&GZ1u2MDIX*cDEXR*mWWP?h$KC6lU10vMGW&;#*&?|q$8Xcq`UG4{2*OwTHh7p z`ngI=*@v0?^vh>3_|Nmf7{{RCuE=rz$P_vW7{3*K51pE++*q z(^~J34>SDr#m25C`Y|YyBvcT1GnKAzBpa~TWI%;0*lC=2uL&cG4?cRPV{Lvf^WlE2 z(UOTOYn)fIHeP`qW&VO4)UmtJ^&=29IU#4Xq2~Z~?KEWMHcsO&$O^f-C zLi`l+)&96yxpF*{MZvk+#0@JpP}@S$0ue*EPK0EK+Lb#TxDJ&51kNsVK)BKy_?8Du zc)$Z+_#!wZNgzVef-$tbXaO%GwTf0dC~n}sLxTXeEe$>fWswFz0K$<3gu4wLF(iYa z!hz5I(-G_!20I=SqwDow4*&Qa+r{Ty$M}bOl4jEQ`*!Ry<5f1uUWHLMfj*c@c`QwP zsB)nvGUGoUc725bvS8l2YeGHUvoSR+J(3zuy@jX#|0wk~teF6Qd|FmoS;Q^&7Iut1 zUebttOnvww8q)ak_~9}KfVwv{$S54YZio<4=NWYUJ_tw=@)h}^J%gly(Vsbub4Jez zB?rNQi>N3x*LBnfFc>dG0(afH_jbuij}};ho|;HWErH0uVVrA>8oC-$=aO9>@7(9^ z?Ul@dv$P98EGzT zG@eSUp|nUTblSLiRT~CHav#dZfvUTq1CX3^ZZ zz%=6m6_&we2VxVYnVdn zJrwt)KW5_^bUlyFHVuQ0XLGc>X11nzFB_MWaRt>;_s}q(*HDFFRUd&cZn~E#q@q~^ zZ17Bx@Aa;Zp;zX~ZQbvsH-AaIzAa+VhP2R_yD@=$q_G1Zje)Jb@fC5 literal 0 HcmV?d00001 diff --git a/ui/front_page/plots/__pycache__/loss.cpython-310.pyc b/ui/front_page/plots/__pycache__/loss.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..306eb689482f8a226045ee85e3e763c7b4b4bfe4 GIT binary patch literal 1949 zcmZWqOK%%D5GHrYeMxfV#BrM-eE>(0R^i&jmrA;q|P#`%iS`h0cC9k^=nY#jP zVT~5Bft>mada;lGOT6}!ztB@>xR#|RTyQibhr^lqW=8dB6cHGI+=dIz3Hb*Xy94&f zw=kOuh$NC0q@)R@$UB8ox`}Ifw_qhtxaC>lm44z|o)iQ73H z#GoUBN%E51(iyzKo7GZj69|!4c`Jk&Y@P%!j5oJl8h_cl6*I#cRm_dIOtq?79l}Ez zVZDVw2$2P3~Wm7cEh>0H-UE0%CD z#Wz42Jf1Ai^$3&>VK(U0kRH(W{>HM&#@N_L!z%y=43C5;Yq=_r9}020N{jwPd!Ru( zNKmO6-PxI`q7Xtq!UIblB6o-+05X7S9}mb!)b6|kDk{d&d_dtPu56OK*-$56Vwdi6B?V<~~5&_r(bvt^^ZU}H3G1llr z>}d?7b$+Yf3{ojYvpT}C8!+Z| z@C=2V`z?*G+w$~s!AGG z00MpBnN6NhRjX3zv{fcZs|pn=;}vRQr58Gvkb;;phRL)|6Au+A0MWi=++3b%G-;<4 zZKh3N54~s(fG|MAgX#SFz!}r0Gy=LF?_z_tKq(=D`U517k>J0|TOB}uihHPCKS8q1 zhHVGHf*ixNxe$;iH1NirEz?=NXC7?3@#L-<+XiDEukuM&m&>|>?g)JhOcH$$)tLLn zO^-5-9nI{QX^Y22el)q0KvT`59o1XbdSD)ccDw^M*6RjF`*6#)r@o17ow+Yr6MWTo V#;>sk8+2<3l{<9oN5{?~{|_-D!Ycp( literal 0 HcmV?d00001 diff --git a/ui/front_page/plots/__pycache__/predictions.cpython-310.pyc b/ui/front_page/plots/__pycache__/predictions.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..56f560f7b0f49bcd271581338dc02ba3d6cc6c34 GIT binary patch literal 1926 zcmZuxJ8v6D5Z=e#@ko)DEIYP@I6x3Lxd0LSAjr!B(+LI)q*x$09B-G@Dfja3F_r|T zk^ook(j;l9bowdz3)`j$^cPZPW=|ApQA_OH?1Qs2-+VLGV9+Bl{{5{x{mUcdA6%>t z9~R%hG(UonMADoTR8WdK%UQt%cRJ6#!WX{Ny*wy7qC?3KMEWwgATm(&oQn|F9T~zp zR2{|7J<*ljS0w6PgKZK~+kKf=%}6JOf(8HfyBF{YM%i?xaifE$jnXOJULX8sVV;4a zh@g@P_LfYjq>^2Pg1;pm5gxq6?eR zQPQwEIp*hdPGV0IGfJPjrd16c)Uy=Vt!&+CzDs9X#SBWmOayrb4WU4;bDy^nvcBb@j zmJKJmDx0{5gCSZr@VDD|59(8m+3vzL80QY%p=Wz{ygOL>4WU2`C1e9)=*4kS$ytti zKaO9|lDs`}7VY51*Hq~Oli0J`d#h0}U-=X0! zK=dELGzdnY_Gq7;eRv0dV8?9=-t!zi#&Udt21X!&i?{5Q{2dfx!=!gkzb8L!|B6fn zuFlz?WX`0&2co@`uF>!5AHU=H z`)eROJ2?eJ_Fz{8Sz(TjBkr)Uo=lp=@=TtrQ3DaN9)kD? zG=tpO1wfu5C{OPJ1#hMlZY>Au2#N!w1_eW2&?#}LM4helV=n1FBvrEIDeUl#9dBdD ze}88IltT-y`~f=NRQ1RPV`xnkz?Z8@16a6Lz%_cf5>e1X)-$S!PnsjE5sV7L2~t3pYGoLpuRu42g(&&Hvq*R z=m14iEuhOf&`)6t=#6=|KHAXwQ_%0`w5Jt1jrn>G#|BM4xZjc0&8Vx92Kq6I4^Uty za0RE)isO_147DdHRtT(G3|2hWo6UIxBmx4V^O+h*0!9LXSaoK!!O(J`4uw4 QU~A|@{X?eM4twJN502@xng9R* literal 0 HcmV?d00001 diff --git a/ui/front_page/plots/gradients.py b/ui/front_page/plots/gradients.py new file mode 100644 index 0000000..eea3449 --- /dev/null +++ b/ui/front_page/plots/gradients.py @@ -0,0 +1,41 @@ +from abc import ABC + +from matplotlib.figure import Figure + +from neural_net.epoch import Epoch +from neural_net.neural_net import NeuralNet +from ui.components.plot_figure import PlotFrame +from ui.plotters.plotter import Plotter + +class GradientsPlot(PlotFrame): + def __init__(self, parent, neural_net: NeuralNet): + super().__init__(parent) + self.plotter = GradientsPlotter(self.figure, neural_net) + +class GradientsPlotter(Plotter, ABC): + def __init__(self, figure: Figure, neural_net: NeuralNet): + super().__init__(figure) + self.neural_net = neural_net + self.axes = figure.subplots(1, 2) + + def reset_plot(self): + self.axes[0].clear() + self.axes[0].set_xlabel('Neuron Index') + self.axes[0].set_ylabel('Input Index') + self.axes[1].clear() + self.axes[1].set_xlabel('Output Neuron Index') + self.axes[1].set_ylabel('Hidden Neuron Index') + + def plot(self, data: Epoch): + gradients_layer1 = data.layer_dl_gradients[1][-1] + self.axes[0].imshow(gradients_layer1, cmap='coolwarm', aspect='auto') + + gradients_layer2 = data.layer_dl_gradients[0][-1] + self.axes[1].imshow(gradients_layer2, cmap='coolwarm', aspect='auto') + + def plot_gradients_histogram(self, current_epoch: Epoch): + gradients_layer1 = current_epoch.layer_dl_gradients[1][-1] + self.axes[0].hist(gradients_layer1.flatten(), bins=50, color='blue', alpha=0.7) + + gradients_layer2 = current_epoch.layer_dl_gradients[0][-1] + self.axes[1].hist(gradients_layer2.flatten(), bins=50, color='green', alpha=0.7) \ No newline at end of file diff --git a/ui/front_page/plots/layer_weights.py b/ui/front_page/plots/layer_weights.py new file mode 100644 index 0000000..7220869 --- /dev/null +++ b/ui/front_page/plots/layer_weights.py @@ -0,0 +1,39 @@ +from ui.components.plot_figure import PlotFrame +import math +from abc import ABC + +from matplotlib.figure import Figure + +from neural_net.activation_layers.activation_layer import ActivationLayer +from neural_net.neural_net import NeuralNet +from ui.plotters.plotter import Plotter +from utils.matplotlib.utils import mpl_matshow + +class LayerWeightsPlot(PlotFrame): + def __init__(self, parent, neural_net: NeuralNet, layer: ActivationLayer, rows, cols): + super().__init__(parent) + self.plotter = LayerWeightsPlotter(self.figure, neural_net, layer, rows, cols) + +class LayerWeightsPlotter(Plotter, ABC): + def __init__(self, figure: Figure, neural_net: NeuralNet, layer: ActivationLayer, rows, columns): + super().__init__(figure) + self.neural_net = neural_net + self.layer = layer + self.axes = figure.subplots(nrows=rows, ncols=columns, squeeze=True, + gridspec_kw={'wspace': 0.05, 'hspace': 0.05}) + + def reset_plot(self): + for axes in self.axes: + for ax in axes: + ax.clear() + + def plot(self, data): + weights = self.layer.weights.T + n_neurons = weights.shape[0] + n_pixels = weights.shape[1] + for i in range(n_neurons): + row = i // self.axes.shape[1] + col = i % self.axes.shape[1] + mpl_matshow(self.axes[row, col], weights[i], int(math.sqrt(n_pixels))) + + diff --git a/ui/front_page/plots/loss.py b/ui/front_page/plots/loss.py new file mode 100644 index 0000000..c27c0b5 --- /dev/null +++ b/ui/front_page/plots/loss.py @@ -0,0 +1,40 @@ +from neural_net.trainer import NeuralNetTrainer +from ui.components.plot_figure import PlotFrame + +from abc import ABC + +from matplotlib.figure import Figure + +from neural_net.neural_net import NeuralNet +from ui.plotters.plotter import Plotter + +class LossPlot(PlotFrame): + def __init__(self, parent, neural_net: NeuralNet, trainer: NeuralNetTrainer): + super().__init__(parent) + self.plotter = LossPlotter(self.figure, neural_net, trainer) + +class LossPlotter(Plotter, ABC): + def __init__(self, figure: Figure, neural_net: NeuralNet, trainer: NeuralNetTrainer): + super().__init__(figure) + self.neural_net = neural_net + self.trainer = trainer + self.axes = figure.add_subplot() + + def reset_plot(self): + self.axes.clear() + self.axes.set_title('Loss') + self.axes.set_ylabel("Loss") + self.axes.set_xlabel("Epoch") + + def plot(self, data): + losses = [] + for epoch in self.trainer.epoch_history: + if epoch.finished: + losses.append(epoch.loss) + + self.axes.plot(losses, marker='o', label=f"Loss") + for idx, loss in enumerate(losses): + self.axes.annotate(f"{loss:.4f}", xy=(idx, loss), rotation=45) + + self.axes.legend() + self.axes.grid(True) \ No newline at end of file diff --git a/ui/front_page/plots/predictions.py b/ui/front_page/plots/predictions.py new file mode 100644 index 0000000..04e95eb --- /dev/null +++ b/ui/front_page/plots/predictions.py @@ -0,0 +1,37 @@ +from ui.components.plot_figure import PlotFrame +from abc import ABC + +from matplotlib.figure import Figure + +from ui.plotters.plotter import Plotter + +class PredictionsPlot(PlotFrame): + def __init__(self, parent): + super().__init__(parent, height=32) + self.plotter = PredictionsPlotter(self.figure) + +class PredictionsPlotter(Plotter, ABC): + def __init__(self, figure: Figure): + super().__init__(figure) + self.axes = figure.add_subplot() + self.clean_axes() + + def plot(self, data): + self.axes.imshow(data, cmap='coolwarm', aspect='auto') + for idx in range(10): + self.axes.annotate(f"{idx}", xy=(idx - 0.2, 0.2)) + self.clean_axes() + + def clean_axes(self): + # Remove axis ticks, labels, and spines + self.axes.set_xticks([]) # Remove x-ticks + self.axes.set_yticks([]) # Remove y-ticks + self.axes.spines['top'].set_visible(False) + self.axes.spines['bottom'].set_visible(False) + self.axes.spines['left'].set_visible(False) + self.axes.spines['right'].set_visible(False) + self.axes.set_facecolor((0, 0, 0)) + + def reset_plot(self): + self.axes.clear() + diff --git a/ui/front_page/sections/__pycache__/model_overview_section.cpython-310.pyc b/ui/front_page/sections/__pycache__/model_overview_section.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ebeb6e954f48c8447f4047f7cd9d32dc75e0af76 GIT binary patch literal 2916 zcmZ`*OK;mo5Z)yzilV4T9M?hPpmq96)znhb^l>SQCQVxeNYtVUT0lJzG}3GQ?nuKimTT@;e|n8Bj)G>6Dv8<0TnVW-#-GH?Ww+Dlf>O zQZ8{|Gl$inU1ct-!`pdDEHbFUay`{pnpV|<@z~FCNnu6LR0`B_ZWEro3qq1T64NPV zFz3EO$<&w<-kF?VUSW)3k8Rj`bE^KC@bg5P}$=^@Klx|8q4Fo zc@qc2{59VXlOWH1|A-tv>WxJf^HAo!ga;yx((PU}3by&C;9<6%MiVZ2MbsOLES3Hk z9K9TRM_HQp^z!`q&it9ZJKhy_*a?y{N71IX{(E$?wv_emeCHvBZu6f33g-PAc=EqN zNX$6Uc1mIcvR>wz#Aa+UlbPVHFl*lgEJ$p}RYnRTu7lNqF~EdWgo8Z| zu?{bB1;oVriKDCRbzIYvAQO(8Fcg*9$m9B~BicG`Wyl?6-zlWbQgHzS4K84ql9T?H zw&O~|9c4cH;m4ABZmLhYpK_@x_k%59uUE1J&sLEuJIvByv@KR3pmIwW&Zp`q^$9F; zfxA_$(^zmoh8b+Ch0AE`W$5nZD#ROLF2x#ZPy-1rILGa;8mrVZ+H{3BjTK{sdK7if zXo7umX_<~E$Ljz2_$Pz}{D6^~2kW3i2?fTjOW-526zc+@3{PwTGl8MNPGB7PZDb_a ztH6qqVcHKK2UuA$7DBc0lX=+HtH8gZkyo=&P6z`*W4+Y{VK0t9D#m6WxuEP0d#Gc*SEVgzfpF0vfSK4uN=2(+AZ|s~i zptQf%zvBrkvdYMEJi+p@vf?aCOT0CX2#jjO4G<(@k>FjEWO=UYlIIc*E*rXcm0|eRHIsMGZKm<=^?ov#-!A)P=V7@8t3_ z9~SSmC05{dg4ecY#bZ$L+6JtsK9)2#H9ptS0z5(N!4@O>__k*VqYu$P&*$g}p8&$n z62i3KsRcfe2&5c(V@Dkoy`c=uW#YWv6$iR7qYN^ein2w?YQ_o zkp?5~`^xpPS{DiGp6~xw1WD-;m>hjc>tZQ-*wjZE=T(i_2SUh`1U^en%j-q#r)#VY z8s1Pj?!h4|)TugaPP^R{=;<`X4S1=l+=)^Ncw^~M7m(-OQ5pefqg6Z@_q&xo*>7EAAH--OHe?^wph^w!m?*ybouWeyFdSkd9<$PHt T>MLZ9&bdPzxFt#!t{MLUbCbyj literal 0 HcmV?d00001 diff --git a/ui/front_page/sections/__pycache__/neural_net_info_widget.cpython-310.pyc b/ui/front_page/sections/__pycache__/neural_net_info_widget.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eb01bec9e6d2a418f7e72bec8cb30c3f7fcb71e4 GIT binary patch literal 2303 zcmai0Uuzpj5Z}FjI?0l3$8H`%8cazcf<|haw1H4Uoun-RSEX?wa8Q=>?aDs+ba&a? z!;!F_63A0t`f399qoLoU-(g?-Q2d;?yreUGlH;T{73}QJ-ptN#W@l!VwOS2=zrVfh z{W?sBIYS&+e-ESh76>PtCM2LRTZuKY1DoQWoj4;ma4GqTaEH4u2zLb?de7~k#_1;M zcqgDs`W>qM^*rBHp%SoO+6cEq@+4L}kHtWWVkbhWvp=+rzv4a+VeAGLr@|gsFUf#% zi`y@nfm7KGT<-DO3lezT=XIECyulY>_Foc*1a-jC)GK{a$}s5*^(Y-=Ph!3;l&*() z&I&{o*~VT)a6KQzHDDBZP7dilJ))HCTl<8;IJA!l#$b~i(|*U%Zc*k!YM-$N;al5*7tx$K>sAyg1!!&3(%A<;m`((OHiAliu& z4!-Id7lo49xMNAoIqg;0WE+@@EaJ8wq0YI za8VS%k|7;hoRn1lRjr3MxN&Q0DQD>J!_^MG14wX~YX{`D&Fxp#(C6SJCK@K$D@gTI z**eJjAlrbs{@R&IH)hfcl{9dTJ7h|qE`#2})HZgT2E(tg)0*1VdAxbPHQt&!Gpuc+ zrB1o_D&dO<4qtk0O&zre_)&6c-{#A&V8xb!TL%0WAez~|bfH~{2*zu8$=Ln#lv_D) zrmnGv5-X?nR)Jf+*d8Vbj?QXdVt&1)YtRcEC7!7h%!(6JFm$iuYA?!?a+GRYW_#Mn zL;lRrao`myj&{crXYJ;_wTZumIS;vig6C-}CiV3wf-;ZBcLS@xe!^jUVt2nDyrrK* zTKZ9#BwJy$8?>-S<1nd;f6{tTO03DXjjSl{g2wGZ=M%XC+Eo=d-q)7)l5i|!p)Iwm zUDL?GhjA{n8>d_V&}|uWU60eeRE)e!bKp}4nn;^o0qVW$S`F%1*jTPv4(wj0 zt1$c-*EU>}I{!>QJ+JAUcE}G38E+I{0W9PEa|r=ofXn~*Uw<@4iATu4 zI9dH1n0yPZ+JcS~PBRiw7_H2j*pW?f&CZ;O8@WbyGH>EXJ|#aA?sD&(a8J;2aArph zPWMUIzXCgCr%QG4U0Lp{SP76@+vzA(+cG{D5~k~WQt&iUX^~fZS)md`-kdGh zdgX~kHZ<^x2&Y3m8q(Q=cVpDM2AxpYb^Tc&V06BRR&7EDu^^XZoBZaTLYyur#^t$V zB13n&mTn)X`LJNgVVZHcqV_pJlEq2ak|7+|-f_xDLdgzH>h2yc2MOL4Lez$t!5Pvf zb>93B&g-3baN_L@&VN9taZ;k_90<`s$wsF0M>tHtoyP-LGtzVmyGr-5%plKV;I^gL* zy9fBfg|p<`+Rhb+I}ISd&M?g~?THsF7|+ z!rSC~{7Cw>(y3A|UekWpkssq>FV2(0Ldp+eB0s`C8}faWkpMd6n9huweAZ4GCr8>+ zN7~ui9sH!dt*KH)uG?juTb_gJ zATcBL`t1TMpVpwN7p2&h0bc z0_eQ4Kesu3K>mbMi~NAK;G!|NES=KMI)`{U7!xd@Z5DKrg=G4RkRK-x)Q-kw{AmU zy_2W%bFlUWvUu9Mq;@! z<*FtgzMgTZ*OnDB5m^Rx_4bvqUPvq<(uEF_z(5Pg1`XW6>AYvw;iurMBZ!TV@>AHO z!)e;P`6n2+(IzZG`jkZumAUGz(%oc0^bj`O7Tq4Qvg)nI$`H3{sBLI&T(4j?sOxg~ Tbsd0j@VS?OLn{PMeqjFxm&*6U literal 0 HcmV?d00001 diff --git a/ui/front_page/sections/__pycache__/training_information.cpython-310.pyc b/ui/front_page/sections/__pycache__/training_information.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..560a8e5bd5106bf52b3cee1b008b3ecfdd82fd67 GIT binary patch literal 1904 zcma)6&2Ah;5bmD8omsD)O#s^%3pV0U6NFbnNJth!k-#7%%Lfob+DmIP-Mg9f?99?L z3-M}mvZb8-1W}}L%meT;edXj=$N}+H?~gZ_gL^bJUES4HHD7&Q<>qFSVLbgS8gxR& z{w8I8cu*eVtsX!~CV9?!9A77QMsCmL><1=Y>0L1CDL(Yh-JUP`&n)(@V9!osZv3Nh znLbWQT_5C_M)eRvS*JZma^?1&XRObqBi)Nu&y$|?FBqH#vVm_fw~(QXF1%jj8S_{# zM7XAD+b};C{ZfyTIx7p~D;wCQD2qXJ3QXVX0HS8+Y{+L^BH*RN+02%>v_F(yoDTn29IG8l>ev7x-#g61pc}@6&FZiOQm+sJ&z#Ez^ ztqd-Gjw+DNj1B!E%KY4u;R{DbmsE=eRvS3eT%6gEpaPvi&6;1amH*Z){fBZB{^!0U z_>OMjd+V0In`^!|;Cti$d|MvfIne~%H2Q9$Em;p^M-%cUNQ(4vsblAgYH;;72_GW% zO^}xPWKrhc)^(t;;X~BWt{rd_{Yg3l^g!6@di4Rh9Au-9LhzSI)Xdn5nnas2M#a%2=Zg@ zT1u}k;lEmx#}0}1O5CN{2E-O#+gm)o%{@H9Kl|=351|FzfwoJUbFJ<3>D~=GuCC4- zd?WzqrdvMtFsZ>v)P+B1FPw7@4uO+HmsY?=2fT2>JMVJ7<1cms>0@WP3NBjv@Osr& zG&q^w|79#u%wo`4-mebyq=PHA=ok7uB%$|7P=)mS5OHMuvCbB^d=4-wUG%eo{)mhS zXZLDghfSHxy|J)2sy6`d5+h{XfZr^!2Ih~(fN_&sPlM3r zHg}#Hp;P%6y4>S+v}@ew4U`@acoSuvxA+oDpSSrk%Epg|ZG-{V&{0d;m~T zaqAhQnBtFzzPj%7(dVPjKJII1jX=w$mMysiEpyb?a#@e)&}djixogAXg&kvm<(WbD zu_C88YkgJlBwLfT7g*Qd{m&a8U-|p@cVE8Sy{MdWI!T1`q6m62ibM^| zQkRQ7noc;@ALV*mQAlVE3T4X2%Dtb2R$T9kgw{7rySB2+BpWDq5{o34Is~E#Q;1qUiuzexmQi$I z{M*}@h++hclp9CBZ=~aRnA{LazZm9-gV>m+8v_B4q6s=S$_i2C#Ga$)GR&R@;=fA+5U3xg*%sEQekM^w3F#PDDI_=@d_{vqZBq0>pPY4|$ zXYe9pC|mX%W=LQaNU|wwC}Z{x3Ux*Tk_AbO=QBTIoE8V-h$>b-V-);j;!=% zje7*&HG=Og;UQjvf_F%)dgwqX_T7P_+=BKbB- zh;T!lUyHU1WHB6ONyT&N06)viF$|MC*VRkmubva9%3 z*$U(emF#6UyL=YI$65Gd5dQ#!*hqj++(e0^Yifw=bKnwm*=1w5B3`0NhYkk4-2gd3 z9An&3jn}xhrwMq|kO2EG>SS%du5D{86$x)1jqn#Ex{co9e|R)P#GCL)yaAzDBHlt7 z+M8f0jiK4{W+Ej>ZcT0{YdYrQZK!m8@d}Ob(=rm%oM=$Igdc*C7B;nFw4H+O#1ruj zb?ChS`tm_KRL(@Cxm0zqY_%=o7U`Vc1m#wwy^gMuFyVnoDE1xZnU`3{oUNXXe>LE_ z;1jAegM5aH4iFQRp~k*Rr%Zl9S|gl0*c9RlI^k*qoXvZ*4an?~M`s@en9jNRsbu!L zrg8yh;$a&_Igg3>0F!`&5kgg$+u&u6=qUH0uHuNI1W-#rWfi3Xj;eB7Gc@dJCGinV zoEH-D9y%ygC8e7OUcF+1UCr5*vynKN2hYU=r)Cw8HB>4_1=K987~C3BBR|MV8IB&R)29bg{!6fZ1@C=! z!lzm1YdX?TK@*g77bA4|P^Sb|P6gW2&=Oyui@uUvqd3l9JL}q+4SFt!3?s?Of5-%p z@B!;p2tF?sH=*=^v}tWNo4tC@bPFDP;bh`V3>ReR#Z$eH-X3Xu1##{~3;OFe>9Ul^ z<0OidA4RxeOhKQK3Zm$nX`EF(f&in>cTJ2HJPC>w04R`x7ARN9~LX~#~i cNUx1nq`ccz= {layer.output_dim} neurons").grid(column=1, row=row, padx=10, pady=5, sticky='e') + row += 1 + + button_state = tk.DISABLED + if self.app_state.model_data is not None: + button_state = tk.NORMAL + + tk.Label(self, text="Accuracy:").grid(column=0, row=row, padx=10, pady=5, sticky='w') + last_accuracy = "NA" + if self.app_state.neural_net.last_accuracy is not None: + last_accuracy = f"{self.app_state.neural_net.last_accuracy * 100:.2f}%" + self.accuracy_label = LabelWithRefresh(self, last_accuracy, callback=self.recalculate_accuracy, initial_state=button_state) + self.accuracy_label.grid(column=1, row=row, padx=10, pady=5, sticky='e') + row += 1 + + tk.Label(self, text="Current Loss:").grid(column=0, row=row, padx=10, pady=5, sticky='w') + last_loss = "NA" + if self.app_state.neural_net.last_loss is not None: + last_loss = f"{self.app_state.neural_net.last_loss:.4f}" + self.loss_label = LabelWithRefresh(self, last_loss, callback=self.recalculate_loss, initial_state=button_state) + self.loss_label.grid(column=1, row=row, padx=10, pady=5, sticky='e') + row += 1 + + def recalculate_accuracy(self): + self.app_state.neural_net.recalculate_accuracy(self.app_state.model_data.test_inputs, self.app_state.model_data.test_labels) + self.update_ui() + + def recalculate_loss(self): + self.app_state.neural_net.recalculate_loss(self.app_state.model_data.test_inputs, self.app_state.model_data.test_labels) + self.update_ui() diff --git a/ui/front_page/sections/test_model_section.py b/ui/front_page/sections/test_model_section.py new file mode 100644 index 0000000..4f85a3e --- /dev/null +++ b/ui/front_page/sections/test_model_section.py @@ -0,0 +1,42 @@ +import tkinter as tk + +from ui.app_state import AppState +from ui.components.digit_drawer import DigitDrawer +from ui.front_page.plots.predictions import PredictionsPlot + + +class TestModelSection(tk.LabelFrame): + def __init__(self, parent, app_state: AppState): + super().__init__(parent, text="Model testing") + self.app_state = app_state + self.update_ui() + + def clear_ui(self): + for widget in self.winfo_children(): + widget.destroy() + + def update_ui(self): + self.clear_ui() + + self.digit_drawer = DigitDrawer(self, 100, 100) + self.digit_drawer.pack(fill=tk.BOTH, expand=True) + + # Predict Button (converts drawing to 28x28 and shows the array) + self.predict_button = tk.Button(self, text="Predict", command=self.predict_number) + self.predict_button.pack(fill=tk.BOTH, expand=True) + + frame_prediction = tk.Frame(self, height=200) + frame_prediction.pack(fill=tk.BOTH, expand=True) + (tk.Label(frame_prediction, text="Prediction: ") + .pack(side=tk.LEFT)) + self.lbl_prediction = tk.Label(frame_prediction, text="/") + self.lbl_prediction.pack(side=tk.LEFT) + self.prediction_plot = PredictionsPlot(self) + self.prediction_plot.pack(side=tk.BOTTOM, anchor=tk.S, fill=tk.X, expand=True) + + def predict_number(self): + inputs = self.digit_drawer.convert_to_array() + raw_predictions, predictions = self.app_state.neural_net.predict([inputs]) + print(predictions) + self.lbl_prediction.config(text=f"{predictions[0]}") + self.prediction_plot.update_data(raw_predictions) diff --git a/ui/front_page/sections/training_information.py b/ui/front_page/sections/training_information.py new file mode 100644 index 0000000..b8f917d --- /dev/null +++ b/ui/front_page/sections/training_information.py @@ -0,0 +1,43 @@ +import tkinter as tk + +from neural_net.epoch import Epoch + +class EpochInformation(tk.LabelFrame): + def __init__(self, parent, epoch: Epoch): + super().__init__(parent, text="Last epoch info") + self.epoch = epoch + + self.lbl_epoch_training_time = None + self.lbl_last_loss = None + self.create_ui() + + def create_ui(self): + row = 0 + tk.Label(self, text="Duration:", anchor=tk.W).grid(column=0, row=row, + sticky=tk.E, + padx=(10, 20), pady=5) + self.lbl_epoch_training_time = tk.Label(self, text=f"{self.epoch.duration:.2f}sec") + self.lbl_epoch_training_time.grid(column=1, row=row, sticky=tk.E, padx=10, pady=5) + row += 1 + tk.Label(self, text="Loss value:", anchor=tk.W).grid(column=0, row=row, + sticky=tk.E, padx=(10, 20), + pady=5) + self.lbl_last_loss = tk.Label(self, text=f"{self.epoch.loss:.4f}") + self.lbl_last_loss.grid(column=1, row=row, sticky=tk.E, padx=10, pady=5) + + row += 1 + tk.Label(self, text="Learning rate:", anchor=tk.W).grid(column=0, row=row, + sticky=tk.E, padx=(10, 20), + pady=5) + self.lbl_learning_rate = tk.Label(self, text=f"{self.epoch.learning_rate:.4f}") + self.lbl_learning_rate.grid(column=1, row=row, sticky=tk.E, padx=10, pady=5) + + def update(self): + print(f"Updating training data for epoch {self.epoch}") + self.lbl_epoch_training_time.config(text=f"{self.epoch.duration:.2f}sec") + self.lbl_last_loss.config(text=f"{self.epoch.loss:.4f}") + self.lbl_learning_rate.config(text=f"{self.epoch.learning_rate:.4f}") + + def set_epoch(self, epoch: Epoch): + self.epoch = epoch + self.update() diff --git a/ui/front_page/sections/training_section.py b/ui/front_page/sections/training_section.py new file mode 100644 index 0000000..2b14793 --- /dev/null +++ b/ui/front_page/sections/training_section.py @@ -0,0 +1,79 @@ +import threading +import tkinter as tk + +from neural_net.trainer import NeuralNetTrainer +from ui.app_state import AppState +from ui.components.number_slider import NumberSlider +from ui.training_page.training_page import EpochInformation + + +class TrainingSection(tk.LabelFrame): + def __init__(self, parent, app_state: AppState, on_update_neural_net_info): + super().__init__(parent, text="Model training") + self.app_state = app_state + self.on_update_neural_net_info = on_update_neural_net_info + + self.batch_size = tk.IntVar() + self.batch_size.set(1000) + self.batch_size_slider = None + self.learning_rate = tk.DoubleVar() + self.learning_rate.set(0.0001) + self.learning_rate_slider = None + self.btn_start_stop = None + self.stop_button = None + self.training_information_container: EpochInformation = None + self.trainer: NeuralNetTrainer = NeuralNetTrainer(self.app_state.neural_net, self.app_state.model_data, + self.learning_rate.get(), self.batch_size.get()) + self.create_ui() + + def create_ui(self): + tk.Label(self, text="Batch size:").grid(column=0, row=0, padx=10, pady=5, sticky='w') + + self.batch_size_slider = NumberSlider(self, self.batch_size, from_=100, to=10000, resolution=1) + self.batch_size_slider.grid(column=1, row=0, padx=10, pady=5, sticky='w') + + tk.Label(self, text="Learning rate:").grid(column=0, row=1, padx=10, pady=5, sticky='w') + self.learning_rate_slider = NumberSlider(self, self.learning_rate, from_=0.0001, to=0.1, resolution=0.0001) + self.learning_rate_slider.grid(column=1, row=1, padx=10, pady=5, sticky='w') + + self.btn_prev_epoch = tk.Button(self, text="<<", command=self.on_prev_epoch) + self.btn_prev_epoch.grid(column=0, row=2, padx=10, pady=10, sticky='w') + self.btn_start_stop = tk.Button(self, text="Start", command=self.toggle_state) + self.btn_start_stop.grid(column=1, row=2, padx=10, pady=10, sticky='w') + self.btn_next_epoch = tk.Button(self, text=">>", command=self.on_next_epoch) + self.btn_next_epoch.grid(column=2, row=2, padx=10, pady=10, sticky='w') + + def update(self): + if self.trainer.is_running: + if self.training_information_container is None: + self.training_information_container = EpochInformation(self, self.trainer.epoch_history[-1]) + self.training_information_container.grid(column=0, row=5, padx=10, pady=10, sticky='e') + self.btn_start_stop.config(text="Stop") + else: + print("Setting the epoch") + self.training_information_container.set_epoch(self.trainer.epoch_history[-1]) + else: + self.btn_start_stop.config(text="Start") + + def toggle_state(self): + if self.trainer.is_running: + self.trainer.stop() + else: + self.thread = threading.Thread(target=self.trainer.start, args=(self.on_epoch_finish, self.on_update_neural_net_info)) + self.thread.start() + # self.trainer.start(self.on_epoch_finish, self.on_update_neural_net_info) + self.update() + + def start(self): + self.thread = threading.Thread(target=self.trainer.start) + self.thread.start() + # self.trainer.start(on_epoch_finished=self.update_training_data) + + def on_epoch_finish(self, epoch): + print("Updating the epoch") + self.update() + + def on_prev_epoch(self): + pass + def on_next_epoch(self): + pass diff --git a/ui/icons/__pycache__/icons.cpython-310.pyc b/ui/icons/__pycache__/icons.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9fb849c33d23bf75fe49b6ed195296d557da1b5b GIT binary patch literal 660 zcmY*Xy>8nu5GIe*PXxP7gARRwE<#aj>Sh!zIzPk@n(`iP~e!aOrJ*0&E_QkFN79Y^f9D^W&cBG`3 zp@W{4tmh@CzR$X-kIR^nmXwL$BDyAJDq@jf-V^j2$JvNad<2V8=UZsIN6dL~#RWU3z_ zWu9mc>9`u@;9_jI7gBpvmwfqnbhKE0)}F^@$17!37%FFukCk0H?k|_D?VF!OPw>{m zyJFO8Eo*CvPF8xrk&D*v^H$4RHG>B)&f21_)nFduY&>__NvA4t;=3urm>5Gw_i0A; zGd$Ml|CygCY<++Si@FQA0!sy(quRJ5UtYX?V$t#x*ILcSp~>Oy6~55VF}URKha3Wl zi^5Q0kDV~}SF@+_jIWKC?0{;|o@Xwxn|82LJFpwaju(eP(jZ&8hvh7&7DUxA@JXNx gk9T;yd^(V&^InOwPQJ%Kju_uh$v#XdhZH8!AMZetq5uE@ literal 0 HcmV?d00001 diff --git a/ui/icons/icons.py b/ui/icons/icons.py new file mode 100644 index 0000000..5b5b7f0 --- /dev/null +++ b/ui/icons/icons.py @@ -0,0 +1,14 @@ +import tkinter as tk + +from PIL import Image, ImageTk +from PIL.Image import Resampling + +icons = {} + +def _load_icon(path, size): + img = Image.open(path) + img = img.resize(size, resample=Resampling.HAMMING) + return ImageTk.PhotoImage(img) + +def load_icons(): + icons["refresh"] = _load_icon("ui/icons/refresh.png", (24, 24)) diff --git a/ui/icons/refresh.png b/ui/icons/refresh.png new file mode 100644 index 0000000000000000000000000000000000000000..e620c0150b292b8116c3f1562f89ceab329af110 GIT binary patch literal 35917 zcmagGd0dU_7e4$nDWRePX(B@dlB7|CP!f$4Qb;8YqCt|{QX*8yR8*8qX)cv$+lGjW zsI5YK3zejuM04+TKb`OA_x|30-g7?Zb9Q?_&pobtt!rItt^0({2J^$6G4gNP!<*;u%|w5T%>nqo z1^wTem)=Hrom#uf)FGtl=cj|^r`=mdd)&HC_@;OzPs^{4S{i+P?p(9?S5@tgPT2qM z>YUh>u~Irolh@95TYKzvtctxw=4&&LzvZ4ON|zs~Ij!XKQT>zEv-qfD`iV+WZ;G)x?*GupXrYc7Zz=GNMAFhul?;&bii0fI*?nWr?(m7wi)>S zZ8JKs&?c*+^s}X6EJwOw!GGaVBj*+qAF=oP;Qfx0A&|Tess#p zoj`q@jCGQDLk3zhm9(Rh%aW!1$-GV8l!Qk2>9mBIVLi#TAA63Cj+Bj!4v}8TtRttB zhRLHg5As=ME)PRVlhd!`C+8jCs?PtPzsTr0Mr1KNN&6qkw)xM)SQrMRQ4P=6dJNJGPYt zdR1T6ihKHE=p5sQdYboU6bih zb)-Lz5B>23`RGhOmPe6~hV;kda#YVBW<{mHWrrVv-I`#w-k8RX8xfV;$*zJKVO_t z-qV;r4>Mp2llOE5$;To3hM&i3Pbw z2SH%T#(AwGBjFbNeX1hAE0f>($Q%-ulm7W%oR@B>PSVzj5m(IY$C8o(h?q>0S!h$b z{~MW#jwu>YGTDZuzVEjUd&mr;pYgxD>7E^Sopi#c7gglx^XK{4?(8FEA0)`Z7?#KC zFqEF4>;5q^d_h$rk>txXa^n2fkkJXDALc#sbBPEhS~7(!=|%Fs4mklFvT>XVWVp_! zF!D&!CN0Kf9DyV;fo>9*nPz_uHggo@QewOc7}uA{;$Wk6z!p(AycC2zE+8LzD+kz{`z%S ztT<&(_rA>F(BG_-h|z=fcOBy~I=K_{qo?ul3ORUGc_jIGocw|mK8smfBE@i=({l@(OoDblx`H{_(OHO>INDOJ-Qzs=6d z&9=KCigq)I?xqBze4C3$idcig1HDp;7QJrick zr5~L$apFYTvVyUWC~*@TgU~;> zw%qrt7WHP{peG}Zd6h_#o=PTQLjUNsVPio!3ZxZ|rDIO6%Y8rj8^R)X^cQ=Yi+%0y zZ7ITgf~sU7{wpy2$Ml4F3&=-WTzKOD=ZFaMk>{A&R}2ndQ%xl3MgK^tXB}Y@@uo^9 z;Y)6In>G2(42Xf!MckRm2F~iI21@qBO{eX;nmNltK4~ z4gB8nC`|8$a4x-z9DxT1x;MdQ;V?SUee$NF&%b0*s4$!v#tJey%9u10G5(P>v*D^f z#xqPGlI@RSY%!#MJ~BGyG`#n3Cg=LlV{_bVHaojf|XeRBlD6gKtwI zX?PypurL|5JpD1I3=*MHF}$*aua546;3_xbp{0m!WksHOqoKVF^lNgMnZ!esh|&&;7rV6 zOp$KF0{ZLIH8bIW!9QNH;?DFLm3VqeGij-NY}I*nG*xo8a>nRK0miI(bLU<%9_=@C z48LM@c+17b+aTo^1HZq%3`Fzg*-3pUU)P9=fVK_tD?;A~hf!Yd^WE2-o~gCQ*49>` zc{3G({WnQi^8AwIf(`8-^4#>p`g7wHjkC>S#5&&Jwl6FD;5;zY+g95gH<@|%XJdi) zhuj@MK0kb5s387K*z`0`4xf^qUZWh$n6Q}Kn@5L-3M0nC(U}8u^yppA72SO+aW;z6 z`9+itv>d+wJYF#eL$nW-D8yoLBmfDPV5oQhb~WETJoLANc|@S8BBDY}PB#rY@aj?l z=7OcWUZh=g30z$IdfEuMD($wy!4V1-QMQC6&)8{!{2^>c&O+=w~KGfqir}6 z?@9yOd@2PvJLj#t@V>vNHK=yQKbueq2hyWJTWzXN^ZmW-hI?C!f;u;mR~`g+)tyl` z_c0e>bc#-#_*T-|@S4u&5VOX7_w;@x6-bkk{4`0)>h6lM?5^hf-au`G-!8fq9v%wr z`dDiJ^5x6oRz8!YkNaOO3jO!#Kz&(|x1~700`}5QMN{+b&!(bvvvmB1o8v!xaB6-_ zF&88)_q~(nR(Z#t%aT$QTnJ%tHVX?YZ!8PyY>BPL$fOk&_ZrAj{Tu)OZ0!4bhg%GSe@d{;sFFM$&%f{2r>ziOyJM4QVo^n~sHPeiuFr>D(^nTY z*s;>;g~450ww3rl4qgaI=3bTfHfo}z!`)4bEXC#GSprJ!bLY*woEkB@`L1J50}f<#DA{kCBoCv?D9JHst}uw%iZ&H1ZHA9pt14ea@{ znv7jqRA%9=f1eLoFWOR=_J_vX{B=3u1Uc6*Kf18J@7H-r$TIBo}H&1J{bZn1FISh$d9&ri>){`&Hq_Tj<6tGfGG!E%`9t<43QWHs{L zU)hCnB+;f=R0fT`n?q+9tL8-S1@dkp#KuoD&rM(Qo{#tXKuK92mgOb znvqc5R#r%wX)5xz59Mg18J+W2uK2)T$b@X}uDPOTXM`A$)o5NsM^% z3CRU}K9=kY^T5>7puKOe=JGkufmU1STw;Sm_Kp=jiA=y?8bx`*8_1A~78cnLFK^kq z%=PbX*`^m_k(&1b`^au`Ql6(`Xp*20(>~KUy!`p8nd$FtS~p_?j9N6U1GE?Unyh{M z+jO$_MoLl?M=dM&d67*BLn$FKao3~a-|HUs)))X8T+**^&u4{xIk5m{ zX04QhQrff1%H)oBH@(~6T*E3~B+uyrGVH^gL`~8vOpfpqgr^_aZqhMvJBAlCZQ8Wgm5LF>OWug1RfIL7GE5;8eWOF& zcP-A$kxHd-O2T%H4i97==>AM*;A0Nd(2=MY;PLQxmHLl&H^tSn1OSM6`HB(4Z_9bC zNn1+y-{?{SsCnhILq24sk1!Sa3_#T?USY-SZ3jNBZp0#5+}rkG(|L8r226i9G{w|-%+)7lGNHv2(M!S}cn|O3HD2Qrm~aVp zKXdYhC0xGI*|1jS@cBarzFiC=W&~%@M1lzO=upep9lULIXc(B&+5J;lP$ssBI8tOn zN-v}qI@!Fsu;?AOWJ5=fa$F(bS$NsrDTjXP1;B*cgCSx#y4PIU44nFo_#g%{QH^5M zhvgBYVZHC|tvlXaGo#0@G)+MvyXLBa_nS0RHJdbQbtSN-vv>FBhu?4COsTuR`Xs&O zrcYaYZf(4~Nmf7bmgR+oYv1ObQug(icRO`?$&Sd|PI+_OAv_@)`+n3H?R+L$eMRq` zS>wJ5^9}vaw}cN>2X-}W1Sgd%4{lEIjNJ{(jv}^;ee!Z&6+N*hQNq~?moDu-Hf5Ga z)kO_!$Oq+ePG_$RIRgMg=|6R)Bozc762-W9TiI^x?fAgYKnVMOHrkI2{c{9{P}LV( z!!>;a)w~3HzIejq8EMd%<4d}|cMInlPf2BDxVJVF&@e0$+shMOo+GsQ^OK`#XQ3R) z-rsS|*+6&y_i5r3-?390$kDL!frw@k=e z9l`FsO~5{?2BwUO;`~jJQ7ae%ZRqS~TWl=Dt#^z6b)M4p>Xa4Df!&|C-P~|BpI(t% zfSDkC0m-&`6gHru4_$DSAvtT7H<4AqPIj;qoG=c7UH4nUldHW=PvL>zuQVzy1$Q;& z9hriIyAErw9oFyv>F~c#QdgBK>-Ysv0e&O{QrQx|FljZ04%I>Qa!h`!m{kZT1S41) zZ#-H82(s4z1gnV6!QSs1G3mZP@2u4qErzz4y}#(;cQAIvzcd> z>)JticC=Jflmhpt8bB`VSx=sjcZvLj_EzhxNRto8$g# z!5`$BoI6;JOdwcdacI$I<{q5FW})#&&F7~SRy2J))cue?L~1@RPfjo49tmozxvE7* zO)L$)veb@VSv%mHyUYDP?rtjdN{8ZZq!?H#qX~#)5{O`s*yiaRt#xJ zyuPGq2GEczKhq@PWs>&Y!M|O1!3(MaOi9UyfX6Wk_|tCt?)vIU_gpJJ2TL)}1D=RV zsiXlhs;Q}2LL+T-a7ZLL6)x><7=)^1ogC85@#Dv(jGeQ#6jgl)_oX7uLqbXjvkYye zf-$3UAWNejK$6SftCKs&Rfp*!{G{emUgGzltGHsMWzOo@_6uA6)^0q*M@9S&G2HX1 ztO&@Q^l8*_!Kmqy`*0b4G=dXX>?>r9l!<+a8|8ab<3wQ|Bh4Mb@-Gwf%ypQ zsJ5l>*2=g?lkYHyCJQhP-6zVOj26<1~H37%vPG2mDW7YQ8Bc*4^Q<;q2H%1u)g=@h5R#K zU)H({ZFa}*MwP2Ex>Js?e6n8A#OBo%y@JrcKW}5W0fN&TY(f$iR}B9?>D7>R`R}h! z-Ugza8E@adh0t8s^ZwQ~oY32tNtn4L<1U1M!%D7c5)5R6VL#`ne$bD^E)|CMwQJYZ zHr`3x!_Ed>vZn)j-j>UXE-e-Czn8f_`3KB^N3Sdx`=iS)8yzl#>w6AUh2|bsTe#2) zdV3@6z{ZO3!y9v4u8O}9pz;n*saef6y>l8T?9Z=H$?JXmPdEELSdW9;?P|RL`|Fq0 za?~im4gR(VU|?rSh8YD<@%owc0$7**M73lppz*lu%P#Wu2YKEGJBFuTaNd3iv~;eQ=q zfMHU+HkJ7ADB4qR2?x%mtEouOQ;fPzIuZ~S&-NmaV(q#4Vb8_*?S6eMtS?gzlsjv3-mlPFW=Fc(F z3gK;U)?|#GnQb;fd+%rGPY(__!uS5#6H=loV4@GP+NQkb6j`>?OqR5Y;bMnW8L&`FVDryT1tiRC9^pQMqISquwl9NioQeucqmz=wN;li z(-jVWok}bs)NPx`WXfe4mJ4v`c0gOr!UIRtZ9|+@R8?OF0{M+=O~&=>*WcgUsslW5 z&PxmqEf}Qw0#EC{?iSb01s>l9McLPRC04{-eJm}=w7R_0K-4*V)4eUBze?NwyfG8+ z*(^?RR{OXJjK;BA=U@wnO-fpEsJKhT|28@1_BS_#e!uMLaK}N_k2U^i_Gwn@G@$fM zoUcKI+I-B9Op;cyF{w4|J(UvjWg;xw$>b$e$mXm1LCc6`4q|U}b~^&fs>B3r$-#bZ zjH&_e0rcLFj+?2QfEBRkFW1kY%1>2$GplVwBAr*Rj3kd?#c_H#bI6H-#MhVXb2Bm! zMtG#!+oTH5ky8mocl3MBRVWwdFF1+yB+OUfvEJ_yI3vu_J9r)z^LeEU9$GJ?D-b*UJdlI1o0`@t8Q^Ec*Wa==BN(2DLj!+@33uY0=QG|Fc{{GZs$U;0C3x-4 z8=Dn_?dh^hwo|Jy4+63YTtKu&E4zp>-6kZ`BGu^dpO*_#Kg1nT3UI$pE>f@%22niFj#QLb5Z}ONFfp!-h_Rs-K!(&g%F3l=9sfg{ zR&V$wsTlI@2A`LS=zyh_RiV5PpW58HtK{_q-z3a7laiKxRy~~xn+q#X7%?EE%V`1^ zY(n^71CXxAwn$B%?r9*({~jLL4dO0p?|IL8j{S!4$*dUp^X>;wT)szjRi8}mah9YB z#_dVeI_*A2QVxZ?Z6EeEY=uqNrq>gIbw$rp$wTwfa zNJvMT2bd);=W`$ud^LRbu{Nim%$r@BY z@qo#hsn6b)?7fI%GP_uS^BfMV71VeU7%Z%qiKw(%22`>a$SVc-YyE2i0}DM~U%FQ0 z%??#DU<7D#_bqWd)zMURS7x#EZgc|35H&i|9npE?qKh4r02z=t1|))Dqwt}Q8yA-y z{JpDLQ$h(Q@SA|3B7#A* zPxmsk85I+A7ph9uFeLYPTWxKSjK~u7fFMHY!5_D8`c#@wfT_2i$I0IXH4`8(fl}Pw zD=X3Ow1u+-9*-6=Ku{1+J-J)`N>^hO8W0TvmZo@*KZ*Z`&JK!qec|T(cA~}LJe=3|-jtnZB5|Fk%d8k1*-QML;K{mg7Ty1NovbQ``Z0=+ zc@kv!`*d@0LL>l)YOtM@j@yPTOkU=7_TW(0ecwukK#@mwS8Ssu^LT|%@Y0x8#iQjCkMRKPwatUN@rrckS%7tiedO)u$S#1n+oIw z#NHx+ahUg5xazXj0-ary#ZBV;{rb3>&_@J4W%DPj;`ZDH6Z(Aej3rDgoC&-(f__PI zM(Z(HLnvvAZN}|62U0ev*tb~?S~ytBX;Zed-r>=KHZ}Xq$vW<68M3mcpT{jD8=uKv zGXH{1|6{PORv-?7Mu!hJ+iu@{-|tgl_(&*@Pli3zi#F(iMXrxNnlcfEY&rIkp3(vzptJcmH$fIxUaa zx$*)sKwB#j!irYgl=b2?5XJFoZ>lbq)oKO=t)ubq&)KdSxnONQl9qW*+xmf0I{l%1 zg3`kf@%VT-gW%VOFcIU*PA2Nuu{|8-kt8ioF>G=jks%VoSO(G>yKb^YliPCynJ!rT zS#+dG$|=F%fZzI5qisO68akeo#Zl<9-B^t~+a4@u+fp5C@J3r|gbTR;VvN+`mW?SK zDciiA&q~$`n5YqN8IJy*cbQYzHk8YgGs`uSHQkQQGziYZnA3pewmb-4gt6G&-u^gJ z7F;RZI?F~rPLPy>MpJ=jd9n`|c+jjH(kaEC}2nKOp;jV4GzG`CjRcbhr)e1^$;gar$|57lv z^asDn)hue(D-h%7Ju~vj0-pz`d z_~S#KvW_oOO^9{IdW1rD2fj;C=4kgVVhPp2<>c{QFVC(f$u>Z_rTdT(@qe`I67z6A zUD4_A)cw_kdO#)O z)A+9!lyK0k3l@M%(E`y?gmsgUk;$|+8i>jlVD=FW(bHPl36aSBNr_lmg=DuP=8<{m z->;?5p;9zXf&>54@vaXLe4m>zXA2gNBO{g4+5YLleOOgDg9+se0UwfZ^xy;5RLe8e zVFVb>HX+$N0LMH)?-#oERz0tMz(akPaYWOB5tijJG`sYlN$E zI^D-~?wmO*3p}dr&dpz6(kS2$mywS-cAYou1>NT3K4}h@%OwFf5Pels6iwc^p&w_( zX!APIE+pcD({Pb!LQoA^DHA0A;K))sZG)_H;7`F6WIzMLO%Q?$n4}+cr>WTY)oHL1 zq?L?Qofl=w0pSJ_3_~;1i3G?SYyv`W{VuMw8@Ou=boLuCeQhvx%<9rmR>7fve&c>0 zXCBN^CMcoD)o4%6DL0MKt6*bQmMu&4mZvfjyeGwLJ~!-pZ%+qXgj{%a>of@BM#6xI zJ~ox{XP+M z6w{l8*~0sGOqscK6$r$e?UadrX+G}{oICIZurnQjh>*9p#v5;fFD@=fMXn_9apzMJ zl1SJoONE^1EH3o@Sn}5D(vru)lQZnWBlN+1s6n(0r6`dQ$K^SMae>*Ry7$*dEda-4 z2vlroX$gk5!zRl~^DrPu)fr`BaUqId7TALM&v|1Ai8(2T_md{gKnnErv0d*%{TR+b zxEYhC#`dk}4q#ZX)pN=*>$=%Xm)iAyueL|pptvx_KaI3A=4>AV zp$WNU6bJ@!OGA3D6N0yEhDAEgZD((-F|qXGlPS4ww1w1#&71Rgam$1#v8nH)Jzpfu zrg_{M7Do*a{@x8Z6V@O`RT2Tz-TFo*<>8@!N$99C?#MDwj&DAe1~l#xWWFW)e`NOV zN!IZBq`-P(rM83Y%u1cW*$qeK4Ix|+wa?y2ar{k7Z*(E{rv*?9v+eZ;E*FS`tHL5w zTp;*&tP;V=B9l#ZxFmA`tCNq%Y>-X&aO7`uu^Sd#g~%^pA9Q5?!6_8K)f?#);zwX- zirtjX$I2|aGgHe$#nAAI@I)%^F*L47nCZrlojZ5FhTG{|X`+Tyg0%!JfUFO>L~%r> zPMsS1`{jZ@F!JhPpq!H_C5F&O?#vaTg<|shy9_4SdHwv52YzyzxD=Ju`yV+fCVywz zJWKy={0t?DWzlU=9v)ARPbEW02_N(V8FMB^lxlz=*u$6h#2N}-PHU`N4o~>1(cvb@ z3hU~D&$oeZo8tJG-?DvNwib9yfk-Kd2tA7;`J^5I=x9!Q|F^0ewZ>!4Gnek-aQPY1 zzyR+Vd;&x=u+b(GlVVx#;5|Qk_6$2CMKVYVz1{_O-n;*s!N=?-yYp)rFS-qeYAJ%j z_vHm{FM+r73A(~?7ZT=(uowu1aR+K6M$?HmtC_HJ%BfE_e!eUf7S#lGTy-@Zk8*ZoN~a+E%5K(jA1EYMfG%P1ix-5aQZCQPboL0ozfg?D z8CR$Y{J`N22)+AMQWC4k3R|3HB=KHf(X$6(mnScvGgVghjKSZ;n2<->kdn@wZ{!W}+IY!|3WYov}l+k@8n{oWOh=L83(R9ArdvAHq zvDoPvH9>+DN1D$z1Q2M~hu7c3u#|U5W|x^L!-1T3rX&B(b+g{dw>?0%;~&-}jSuLK zM0O~ScU&qj%el0UL>q!9e`KCd_kqnA-##Vm_u7ZUzh4nV23m=u6Svq9ylsIz zpN_=Tsn_C-1}%VP49lsK`9S4xoc3UfWnIP1C7Y0DUXvH?iLjLi6nU{5X95`LO55cW zo9rG#Qa5;AGt^Kb5*!RBWVeE6V*e-J!aK=)*UIUQYx+c0d!I^X5HYW_bT&YwzM4*8 zx{}fR&oP%tf&0`BMT!peiDQLMdY1^85JN!Kd@o^-G#HpLz3R6#WM)m+Nr?0?kl z!@o`52#_k0nOVfC8>WW+K7mv>Ka9uiPy1WqPl_*RoFw@<;4X=j3kt$02Oq1LyI?_b zd;9LL&kvh}rJT*MBz*`N{Pq_paYmG#+b&Bf+34MwtI@Q>|nyBi@d5brcG+4uSk7P0s;h?p0u2YF-Ci=Ca2pk)K{Rh484 zim1e0<{^fcl;3KDLIxl}gb0=NcG8C49&EeOp~@qKW`f5}Vw99bf}AFI0L%)8A%#%P zbwmm&!DXG%#2}xNH^8PlKhdf+Oh?|_)egjPDgFz6%D!-6w85F=jtns(UYu9I7~K6C zz(@mve#THHnXf1Pk^2#9og4sQxMrRjlaYq6_FHYl7N6@V$L z1Ejga;lBzg@DeVA0{syFu>ak?`=8-<2dmDRL+4;K(!SlFSc~i}--=Dn-|8J*Z_s_U z54f`eI_l=xr2L6N@arFY*6G&Z17yE=UbVxkhI^?bM4ssEGj3sVS z#RiMm*jPJ6DWH_`&+zlM=r%&yjL7!fG&(#uzQF#{sgGhj$-V`4)D^ob&;3?&b?eZ- zZbtxHWOpHjc}nVrkW0UGD}j~hWCliOe;~^qaf9B@oQkIK)7ckQ?dOjm#fETaw8Uk9 zDVSIbq8U>E79=@bZ6x2n^o(vp6Zc1&lvI~3bEq{M0`Y(!7}7?x=0~DkbIz2rTZ(;d z!&lZKGR06I$lzrmwFcuM@7+eyPO&oHp&I%#c>Y(aXOj3KT%GZ!y$?R*&2W2-tXSWQ z#TKF5XyoIOhqHbUZuvpfEa-t(e>?X-L=wmz#LGpeOmRoTELQFJU{p*;oEE|RBs5Qz zq_7tl!lz!2M9spHGUr9WxnL2#77%gZgt}bi0sb7#Z>50hMd&R{xHw6?vhfNRIVSn? z7&_%IC~BQ>8xc{UBvv3fhY8as)gt&aD=(U$7yP!?Y})6EL8*Pwph zj^>6ZEp9x#eMq}x3sjXay)Sa=+oC;zn21+0BI~RU zsTdn%SBy(5`LPYBrRZu)J@S=qA)76(fGi99Og5_**r2hddk@BQe|)tImqm!xVtRWl z&#So`RCwdGt_Qv3F0YMO`~|#Hcp#~s1_wJHYQGi(D?oA^GD>+1x0Suem)?MLT_X6B z(R!o9Vk@FgFE47O(i~8Q`nT7Y?J;KiRjULgNgh0BLftZMW1*L%d*yXRBj6R|Jt+%& zE>|~so17=v0J~!>$?do?VB>JzhqNPZgC8W8Abj!_+{{az>VxCb)uJBcT2J=oynbZNsC-D_9$0{(k>_A|*8Xlk3_g%ETZTFfkt zN=;7zcm1K}#};InRM4wxVq&?EyO|`fWo45EWHC(e0=hLTS8EL=wpzs z-ZTEq2)+p)9q??Px$VTiZj6@dXe)g?6tLK(d^bsrCLV^~wt23P!eCD?A(7v_3C}PB zOSn;>i7yrJ`b8y3(phC<0{OR+PVu1|9fMaX#CNS z2ivax*W9keQKoE-dPiuYswb@#iX5|cnF7J*XClPZQ{C<&1sm#8!|?!M!I?aj1q6TIpzlY+Q@ zbImt>yfm(sMf4TiUbl=shC*5oVC>tr|2*6Q_f(e5=MW0Oc!F}(1tI@$5y@2@*{ zki*6ih)_4o%gg%?=Brq%)DnE(xFU>fXc4VN$poF$eC{q?Ekm^oY^9f`%P^ai2BH+F zZEtiN$yMqFoktH;UNqhvN!D1011g3x?$t>fagIO_;2V)G=nE#R@M`5o%S4sgNaV5H zL!LCplhPV4AIIl$YvdjewIUya6&&f}ChAe2BYhZ33BhZay}#M_{5 z`YWtGhZQi~zpr_-^Nrnaum2|@@hrEs*H%W4tFA}|^QMI`CU~0BjWC)jNRDx3@+O_< zgnT+OnSU$isuG!k?kp}S5v30f)iUW!1}vi*$*YoaL_!>%?EmZtjju%se{?1P zxU4G9J{$P!qh7O~bM$ye!qJtHFxpMD2$t|;65GA#%~!NScgQaKui4xSO_9)~|9ehS zVG<1Q4q9?JqEZTzpc(xB`wLwFMUpPY*PirAPOF}?GH?m>9q&U8>7vB+_}phtpMI#} z%F>#R8k>0H^l5EU4iiu^p}U1VGrSbDruNE^G5lSX`0~HMFY)?kj!Ty! z!htd~XL^l~Le?r|_|$m+#{aL6vq^T3R*C5NE2#-6$Y`GF9Ifn&z8GURA;SxixjnwW z*;CJg)-^bKH!3>%zjd4-sbD;5%9NsU8$LxeK9$C|)Y!HEGm8J8FDI_uu%Y3Ij>%g`2>RhB1BLD zYHpe}o!tPKu>M^LjM2-=oR%Uqq7JPq`V#a;OzoGRR65Clw7!mnkqjd5l5GCjXP@RR zALEMaVP6jQRPP;wIeaxHtq@lGH__VkV+^a+6$kK(I1~zF zDECHn#prTw>2DogJR)Q-X1MvZp-;w`oZkMY!Gewh%~1+xpJCfMbBN|f z{$#HFlZ~UBLI}D;)=%?g=Wj$OE8}&hwU~mTidcU?yQRS{=Gd`N-G=Y3L-rg1?vIlBv zKXNKOkUW5p(D}r~r+zEZ#Pl>?Kf8WXBg_=;6wvxcR`Uo}Gl|ago&nHN0%r) zSTR1s%8>896`-|7``8iEDTp`{<61HDH<{LIygU%%*bII472em^m*1VkrZzV^M;F6n zNDO0_4^*?1_hnVLfY9)z)k@eYR%-;%iJW(@26FyEju>y~>vx3Q5`m~aUvZP?RER>t znQl(&krXcRH2Bv?FG%#q(9U}iLx)7;SED$j0kiV0B>NYSqRS?fII@B#JF0FeY;PG& zVYxLDbYf2b%*$$vunLj$AcYR}j=>s^Rz_9s=%0h%MO?sJjQQzuRgvvvs9?P(+$ckm zKz;@XX#f?oHiRI`_EVMY-YG~hMl0e%e0l6#z>|sbbI&i_To=5?Q4b~EsNG=gfxTrY z5dt89!{o!!od+rutX5k~T-Jy+R~4zoU01Trda-pPMu2Aa`}gN}&B*>;+YuLdnq`j= zH(+6m8_}eXJqeeuTvAhXOb6rVfVm!rGcJGOQa5A8k#{_p8u2`;l8@f`$1xM*$F_MZ z@&TNL4}6=Bob3D(G^At!2N@ifPo@7=Df%$~LQZ`=P&Rb)iy`9gMZN&04YoX18v3?9$aog$S`UUu}6j^YuzBS z)7iUBdD~4Ma=w~tI!TEr)-eyc84$4TcmEzvA8o0ZO==uxJJcIH)rHQanI#vNl zu4LWC#V6Jicypwi?HW-}tT2Y{Af1AWZ`Gws>w-Pl1E3vf3}vFf?@DzX!lur>sjdY) z1QkQGi=N0n>LU&YQO%6NrraGzu~kUoN-3KdA2@PMu!RN(68*4wu3!PBrcULSHjJ&| z?n?l<(}DQK!o_$~#0EK$9&|9JWCNU(?4b&vzo#>v93Va#{|ET<(j>@ptTR!=b+l}3 zR&?bK-3GuMiIPn@F1`kl>NmVt_T?e_++SaM8yw6=j>=W~xLBkkvkS+?Qb)~D!7Y32 z!Wm>xCxK>ds_9lk80sxoFTFh$<2 z{(LOp3F{napj!A)esJ<@F!VxH^Mc8Sd%E(_-=CUlmCOB^^?TATl0=;$YLhUJYxQ)I^RZexy}gSu2>Oj zMp13eGa?caqxF17@y{kw9iBXDptHyjfd6de6VL5~&EOwNMi4y`f61-){pu0GhalZZ zEFbxv8NC2zA_as~bNHg{r@K*>!W;|%G1t?a zfuLh7GJvrMdqbrbt&YAK+V`W_HC`RTM;CBut%u3_h!L?srkbC@G4&f?M+~M(Dwwac zvXXK&!H+ehu82g^p{gu9Z04g4h&9O-?qYwnga~lhf@b2N{UWS!cL1k#L6dUm}lJ*ZF{Ps!N z_bds0Hs{oyM^yKq86XGa=tAd`gnYNPfq{Xsui_G`SjbJ5#}h^LDdNb14lj~_Tx16i zTlIYOO^{c`uBqN2$Sq)!vu@&SR${+w+T^OVV&A zn#p=68R+gYJ>Y8|ZnuJD!%?DQ#Fit_>cNV33C z9J72(&I_QuFT;JFTzM;AMs2{E@j2&(_U);PTaEC?di%yRbxt93ZzF>RjgyLtagT$| z0ZWE#2Fu=v-hGKYqrSm^vdFR81r6b%(*RyV^+MtRC}Cnp6wI{ReF@ z*}t|Bi;u_-ZSh4y$uYo96z-Tbbkq;zzpU+zNd3-I*8G-tnl_$~bWs0@Bh-adNjo5m zcjZ~&7Gtn{Bx;1DEw}MyK7&|rGKrz$L?R+TRM*#X^DgfrO2|bgO)3~U#My!!A$chD zTur@^G4zjFZ#3HPMq~d-naakWcXj6hmhxJfqiYd&1kruPqc(-67fs$=GB~iS zSwZ|6b)40IJRkSe`})Tm;|k&QT*4u5N3gp%!qwRqpo(N_RucBAj>~y`G}9?>C!KNf zte#xQg{YdY8%uKL%nq+NX-B$RB-ulpqvA9&6A8hzW}`l;;+Td!a_l z3oi`rL8ICPEgDBy^TYw^$q>Q!S0~@Wdf9Pcf-=Fi>PL(sPdfVo%2MQw!<~V74t$R? z-&xfvh)muj)TWNl(XB3HZKAyJ?)SIOIKGNo#hthX$!hR{`L#AOs0dquLl;j0XOIjKQ&cmCx#s3-34`Y>HV;a7caODU@}+I;8MWczZG zSwir!?q`y+XwJwWwMQ_NE)n8o)}Vo!u5gh~O|E00RkXi6$?}HMN4nhaZ85yH+}jX% z(dmnSzH5=lcuf+$2>$lZLT*QXY!e^M&L3?3;McalYq_uvDM(oE|Dz+Ms)eNIu1^mx z=Iq2QB6Lwgd3m{SQrCUUC7*yGv*3CpSnft7HVpq3Al}h**s)csTvu5 z4xVg5%Jgc({c<9B?e(aAuEv!RVSjs--BE&2gqu*1T@f!n2W7hngi|5Pjm|xhVw1!@ z43B>=4h3lR-f!0NQpo*!3r&Fh9Sp~9Ja#DoeD-&9ewYQ9}@)A{SS@uPHHoA^Zv$zK9)^Swa z2uT*e9%Xc*VXFkvFneFdoOY?c36-I6_>ZmY7#<~s4lqKenlT*jdGqHd!SO+^h%rek zheR2zIP$fZ^{&`=D>UKY;6+nWat#a#tJ6Z$5QaLprUwUdx@PphbBkZ`z1Q{0&2Db# zb*N|}*G6MO{sF6FrQZJf^Z>&B_|QRgb9aWD6-dQGl$JHyDygC#qhf6I0YIF@+VKx6VW&R*Ls} z*EOCQcEE83To@g^k-(qx@UQGFvNq}@L4fUYs2bt*Z{P<1HRpW0PooOtZ?pDlOx6j8 zM@HA~kl1hP5Xz{@-80RayqjQyD||d3u=Yk*w)TX8k#E00{aErWR6d{p@zQU!o(2zk z7xJxqkf>0XwOI=sFMDtIynkD{XGXi7W~IJtl_OX0JUxL67RZn}lJ# z{H68QrtDzU>0KsiA2}8Cvo#_{hlz28d-?h~h)oFxQW=6c?jHoe8sj33rBNM^HhHh~ zf2bq0eI^41?n6TtpH7j3jzA^?GRGi1ok-uaZrpbX{_bGWPYap(glwLVEkng^PiM}_ z&Q~cb9-v;zrY`OkGan*@ZBoUs*3mu$3&iNU@v zQL4BG0ALIi$=OR$mQu90<#Ts!5~BMSq+r$%1x0y1>ss{}yFSW74nDFqa7O*emLdaz z#6_VaoCb>S0h5!9H+)9wO5$)>Rsl_M14t)KvVx>~AL1Q@;i`jp+_YiZX1%)kYf1eZ z?tjn^A1*oXs>t4i!qG+B$`T1YP*Nh{HbmgT3Ohe*$d6RdT2Eu_tXsh(=I@2jC_M@0 z<_E#|29OcwTgf#3h+MITs<>VIa0sw5OMoyP5j8BD-0w~o-0=7$dZB>&q=c03(3(Jp)Ya3VZt}1hs^Or=Jq(VGuq0(=e{?h!-0YfBatsr6MDELWf;!|n zaRznvKJ2?de!=AKAStw=MbhsfHjkaXjc_$MJK`vrBQ=GzaVg10K%K=x#6yP z>Ri1H-q#mrp1jL*GjG=Pp8|tXkBrNgiC8Ax8i?Y+Yx$TvTZ--gFZ&%XQ{L>d?r;&j z6_V=%r?WXOGD5|cR z*+g56XQuhflVV6DFv)9vIwKdDvHt8S%}ikErbmY#bg7gagPr_=GaK|;(FI^_1MXf~ zHlEnYPQdLBbdm(%pj*mJd2M3&6fzBKif~C2LC1PiW`Yl{0jGwviH6(u3(U{UzuW=3 z5LQT&sAcFmAM?lBUu#ocNm3G!tbCs^-Vv-gIb9=(Wc^^5P{z&X^n{QbaNq)0+18$j zOTs4Y+BH{k)sOAqQQ72*5@OK&uVL*k=eKSr($!iO_mv3>ix@)^sljDBD9K|>Oq+Hc zi~->5mbf7S$Xet(Frd4!KiP!nxAm7N_c+2_%Kr;%FTwjz%dbdXB-UFVaR+iz6B80R z<;BpmksDB3fhv+&?v;;$EWf?TSJo@UnorUwPlMJauK-3!b}I4Vju-+###6hQQ*;Lw ziN_h%1+PXb%WFDim>EP$E|B*{ZgEv^hO;daMmPo}6~3WwS$Z44nPHeP6Y% zxh*iP8`FN4AYpDEScGddNHS_VPCs)2$t^}P0c3#sCCWI`PvS*_79k*OUOlmL7qi24Z(tuZgQT>QEt@ ziUP_X{cVpJM87lm1boby>w#-JP$00;$H!-T;cm#d%6}Ch=<5yc-eag3*iq6c;KUAL zkKGL>`0a#WKEy-6F;zYH@B0caDb_!56!vq<_Mf<;5B6&dg<8-8-@J_>4 z-~uaU7^nBaQW++w6D$Pz&RA&3r^7AXiM#$t_V1&y(L z65>UfpZ)cq+o%`13n`aUB1nD4g-_{FOB23MM~st~Lel$5vH*SMmyoIr6d*CxfRB6w za?^96Z)BJ5I&PT47(RU9Du#oTVvC{#k&nl4{Oh{l?te$_3RWxudBj3KSg~>*u3xTy zhfDTIqBPc^9^f()uUdlkU%YF)pb%5`GZ3}Ag)1%Kaf9LQnDz$H)+J&j2w80;A9+i8 zNFMU*NQO0)yB&4{wQ+IC(~m~>uFArN3%B1iuwzujw8a9UBq5gwbu1Y+C@w(8Vj51A zaau(PPG}V>#A;AEe=*&MF9Q-tu1qrI8&x=)rea8LKfw)z4&){#6^m;?K^^d79Kc5M z4^G1JfBXn$u$tr(1K)gzi*#_pkbc1%XGCn8)K8czELJTpBiw-mcrI2V6c<5ADJm-V zpGL&a*>SvdOx}F~^F36FN>!ZHTr$Oj-=vXKhloS|wt#@*bR}ux4dD_>X;r(qyHj38 zdH}{C{~-mpi;$WPJ0Blp9qoA*bC#cym`j3)&>U+TCz{{}gq1eux2U2A$y|(ur-s)Jaf|)84_fMdr4wXpDvOYx}6B0n%r{;(p{`fLl*|A z9%>3+KoA{Gw@sLUz!xdXJNmm(T_wJ7tsO$GE9Bxut-$|>B5e$TsLvdqR&2&)B*-P> zdYy!P$u#8A%N1gzUocX_=fYK4gN|`Gl$Qjq8^K^z!N`~n35if9X}GUk6W`R+^|hw5 z#EN0WxCzDKjnQ7AE*xKkFuoqd)dA3Xvg9&Lh)gahoTK;83s4eFVhz&cjVv1{ZYN6R z2hRKelnT<==-1pB5Vrgh{NYEV3Im{1dNnp~ZOz=qH_kK(&ReRvHi z8@O=bHQss)=kyl-6pqjmMy*E}>Up8ib&!&freSb)9f0Z|5Mh@A3Q*|U_ikjh@Ngi_W?mbBPI5)*TWRG5m2Ft&;i*+NXnTGm8q>JUNCtYXW!VUe-R#TG7v zkTxB*;d`V4Hn!VA$$P*wm+bSSc*;>VB1sj_B&8dk( z*(;OQd|%<3cxN?69qk}-tgKmD$b`j9U-kpI0>nQ2E1AojE()|y&K&J=F-A>Mu^+9+ zZ4%tTG7Kf1B&tWbL!)p&$FpCERVc>%+oY{^4*&IrNCiTl&p2~QLs3+Z^KI2LxRDVG z5CKacZSovIHodSiM;Jjw#|TdK(rVhYfWqe?%9rPA8@E#^(-Q{%abb4Bp?I=oNT97> z)442>F~pFmKRjQ*=RmTVR-jvz^nH;)TIm|JI>-J~ z=BwHh9o>REjo>hT#<*v5k0a140mVo`Ma+#Q93G1MNFe{|Zf<@zNG}+Q&**$KpbEDd zMs}S4MuS4?hvbgaVUZX zFt$e6?HSFFA!k^R6;g+t4ZJAU-@Z{MI!7*#Pxx#QvP|V&dRe{?7{3lrJT+SED$Aky)A`A257I>uR1*}5u7rRv4uNQZxd*M5 z+|bm%Z`OnFO^F{0AN;*+p^$|orxm>?{4nv1$$^z<8Se?<`$q15Ff&&+(P@BDZ{ z2X=)tJj#3GHr3$dfGA4wCuE`~qy$7+=ZHWG6DVM{YM>B*kG)iLd*_bnFq&oc5q202 z#fH>t!1Rsc6svKE_rI&r{YV&M1&KYmT=O_b%q?h~@^Y*?{sU%ggn6mZ22m?!mFsL+{6l zN3|$^^=dRd)4zmyiiX16r@(OvS9yEM$VQ8j&04ZWOWGh;BfN7=t`ME=R@Pd-Ajs5Q z3nVJLVfTqw>R@NqK3(GzDv~dCjCPoru}AYFo1qM$61T|aoih#yI03#~657D%9-k$9 z2{{a!Ppw}g#dTcd<=Fbf&o(4a53%5!%4iWZ67_YmL7p=qF2(J% zPK5A}U^ zVm@`w|I;AAJ|)G(=$Pa2dWf_m57Kw~7 zEk%))NDM453T^NU%v2vZhw2YJyJQ^OF01>Ih(3ts6g9kG@}?C@BE2F~5wu*KWoS(1 zM^?KxWVZvqY|(d9AWald#!=^4exX7=Dt-9~X+RTQ4#fYkR^J6@MlL4IE9A3N>|cM~ zUPdP3Wku-t0KE2Q)Gay?MZhA?uvcP5un5of-@WqRZJ^rh5aTx9*07!Wi&)*Vj|Tgg zKVuG#x}T*I$x@YmM}t*;AS#M7C6_23zN1+tgrO3;sNs^fieprakZB=B6KD)NBtV-3 znp~>>6VqdRkoti7zsq6Xl`q{JMK=@8ZsyTM=e}KZkcQCr*zyNlIQv>7Ma)dZJo;eK zOOnhz35aGsrsG_+>?+r6U6g#M0XN(qn=AAuZoD2aZ=R&4WPA_wFC>8`HT2$yJzC~M zsL{>dJf4UFy9sZlqn?qmFdY1N3H0v)8nxm!N(a_`ix#0bAgghBoWej(x0Dk^m|Oz8 zS6Q2-1c9;MS6-tPmK_cu;e+I$^Rksrtd={63MzUgsxu@6P=VWg-I z#x*)&+0DR+=gFo->Ky6BNe{sjh5HYTTzt8{!dxiVV!inP2(%_`p>OU7dJmjvom|ZU zbP&cOPa!&@qNtB~J3-Zg3uI)eDL0fqJ^LViZs8d4BTVNO^7FZzk&x#wJ&lY8 zo&|L1&nucBot~!0<^c`}FM#cTXIVQ%R%g*rB{>`RC^Wh5w{e3WeKu|sZH(^I6Dv?x zT#AC-B+?)SSma+51r(YG1;T?`E!h2+RffYii$EE{4opL5gV`tdu=#u``4Nw`7dkgv z=%DYN?VjHiCK2C}ClztGTDBj%U?}Ysqv?J0KxF6a+y8WFELMr{lF!MvMaUnxSBRo$ zoke~cgpTVd1na=ktN@=<%8PoPdAfLmNRi%*sz=A9(9A?>X{oAyTx3Gyzu1$dG=?wg zt*KN5f{i|a91KD6KFUkOXU$5ctINujok%dkUdh?0Y|@ya-^;r!U3h_}*Lg^=k7?4d7v4mvF&B-MH(T{8qG z*F|%Wf;cIFcK)I9J=m@8M$zH_WjG;lMZnE^h&lzkWUnX?LiX>*G4r&5ODg)rwEgU( z3UO65r+72z^z^{3S^ViVeT_OC8PRf7<@hK}N$(b(Vxox31*j_}{I(>ddjKa)*3HSu zsg!ul6^owXXRlajv?zjJ`dpq$-Zfq#T*?=y-5%lT^8d11y8e@gLb+RX>?vhlL+&c! z#F{cRc}}8!OBqN9`UMM>eG;7yx=62Zi~soTUb&xSY7ToNj*$(q#I@UG^OI7_3uuMX z39TY!QT8AAl-qK^XV8uH;Ttgst#LE#8yS>S)Dp~-&E)gX>-vfdOJ3*b3i4A)Irl_@ zqHi^Y*n{lOIK%M06}G~|aUe#XIn`#tdkPf!z*1>=`fY^b#3;d$y|26Ma(E&EJ0TX$jmqgE6DB@w$NwfQ6yGZ=i61h)NCE{dVzt&1|e+JcfvB$Nfj-pWKk7y6d zq1S6?htFJFG@VAL>R)-7W&}d`BiMEM0L{}uIIk|VRG7qz4XOS5NnjJ_M}yZRcQ%MS zO)R(7Z($TJCjwX@4}^R+I%|}UoUB(=`42dzrx@o?Vzr-`PO7~hOtnl%b7gFT%ntZ? z2Fd1za=!QTlvB>HIK>Anz$y!=BB%1=7q>^h=O5}+A~`DR8q_otqRk2>aFAbl>TMX+ zalV=fxt4Te$vFY7$?EyrX%`7$q&jhT_rTm`ec=Go4^RRF&xVteRBlorOd`;h?I&88 zBvfd|pwgc}u>D@BTS+DgHrH%w8Y^g@=;zlY>l92^de=mn4={P5y|%h=oTU5s~m8st`sq!8Sbe&=5gwG}Gn)5WH_t$%h z%bQLs+)vM&ELf>%r=e}Phfc^NVsXI^`_Kva*4{|5`Z<8ZXhe)$z(Kp&4;N)AG^Rm+ zbXx3ENPAx;%r9~pQZ3D60N&z;*bVNa7H5?jTHWOhu%jLl9*F&X9JNN#7s)+Qt^3_a zkyxR5O!hami*D$WbBzq?RzW4@A;HBFl+ese6zWrmwYCxCpgnx+zrdoFLj0$=mh+?c zAh%%kr{IJ0&^qeyvrcQF>BO|Z)}fGq;a%j`Kqp38z`RzJc4lJ9Gp-(qxt@X$mC@I0 z3U-0|r_9QzQdECLj8S$U+4Xo-JIpH->X&(~F6dq^kJipS#7?b!9Xl4V;qng)3WnPl8GIN0{xr+N2ms41;A;ec&RYd~2 z4spWy&VysfXvc%IRPj*EuB*92_HcXd(ewMESGfijS+V{P=0k7%j=Q8+(_C zpbkY#zOK_4DNYV&)#JHGY119~R9{hZl~m_8%@x&@_j}O%$8#J9)Gwp>_GU@xYMrpI z!zmf=AgY|LGT$aSq9Xv3Ai5ge1}HL1Xvye9REr$R$>3=1kTiDZloPWeIY)O__SE{H z2csbG{WZ1D&kQX9cFa}c-S`ZkF?85{TkscMehSVdo-jlHFPP0CBEM9d&#I08S?l_nzDPg7Ux zhh^83sRkE+R`>c9M`gI6Sw_^BB+6U-dXcYs(L$M30_<2tkLp~yPw|325_w67BuQq` zPrcU&*lwb0!RU|gi$-H{gY+^)m^~lnE~mwv29M(F>KRW(?oxn2sr`akGGxNsgNHKn z%1pQsHOe*E6Smb2oQ)HUfxl1%l!$^%pocM&_MqFN>j!KT{REjB1746PrMT?SQKI-F z(CX=Vk20E_)eOt_k)j(1UvNr9v$m%Gb|y`f{zZrvj@Z*crAa(gA({?S;bA9sG>ZLQ7Y0Y(O4}ic3jEMQ9;QK zVmhIR;3^>?R`cwc3}VtZ7?8OP_@FdU(N~HLxTgX_UJ(c0(V{4z>)2h^a4Q}H8fTrh zvAsLAS^s3h-;MM4nh$#!O789}xr!(fgTwCyR5-}qloWAEr%B<;3{2 zp;f9>4Qq@gD-uZ)tl39nxl*UuXynMOh+k=c#(ojp6oedD*n|4x0#9XU>EQXsya#I@ z>Vxnc4T{dGq(*qzA&Nex4mEXnQYl^~s|2`|?U)p|U;Y(1s8GyxI z&9&DKa2NGM(MSKG2J5CK9QT#LUP?%6sHMd|mANg|#Qa>zcm2kVuS@C&kAUJknbuKo zq(r34dWfO#RJ0#$tmh$>qpLqW+4`T_&mk`_)lFgohe)f^)Pd$}%uG?pl=tUswA@kD z5Z|5(wI0rx1X_mj;EY=P-8E26J?dm~jSNXblG4boyNYU~kxZAF1CM6`L>yM}hSZaZ z&c4YEE~2-Q*rBBFioP=SMPgWy7!tJbXU(f-Pc|t%Zc$c|!8&-9Md!2Y_EZ!hdGgpt z3U0r2M@{ZXqY8c@l@#e5w>EiHqjz;tsn}{4cFRNJ&p=7bHz}=eZR!}#=1Ia6EMA#y z;}#q+loQvR9;!UPk5u@5GYZ;ZhX+O06w#>RJoTFrt-_3em#e?AiEYrI=QOzf8ca0& z4E%UG93OUOv`0t$J;A2PIEzI?ZoP0;PRw9BY~08e3n*+q3wkpDX;0li=@tgYIKbnrmSEwG+Rg zyg1;6+uHG>yiRT90aYOxSM}k=#TP-HvK9ifq~T0=cXJpQChGDO8JFY6zA49LjKR}M z6^%7HgyT3zRvu68xFEA)8~GPe!b5+6l_}=}4WLZqzV(6 z73L=?G^{c#+obUf7!|W~$UP-WY9iRFB}0d8mz$w>Hd5wv;k8cDtY3GBsy&oc-ru$V^t$bOYi{ltvExF4vstUhQR=qiAoP;iDxHMLX{B@0$b)OoH6u&QFG0DJUh7^V+_Qt5Jw*+;AIgs*Ya(6 zP~*72Z)MH|9dQ`j1?7W_c=8fq@c@pWTJ&ld;&pD)TA}JVXse?`nbk{jFY#B~BLb5u zvQnS_QMa-Zpsdnxlu1+i^aDwQWUc^B#vAE0OJ!kGaYkZI_J0~gFz8DwlGKHywfY0S zR0Drh#nSF}>%Rp`Yw1L>COGVK{L)-ghl+EO`q9)l;TPbn)d-0}QCX4e5@8%yZ~u$M*&{1&^ov! zgC>rn&GCSiFFIQpd}4Ayq)!2J1c8Q&*Cg9qSOI1d@q&pP}@XuPjCwiAxe1xN1C*;JU~7vS4ILq zk>l7MRwE4c-E5AE!f9_|ryE6gJz!)lbw!9^OW$NO1}~7IO{$53JS%m3)$Bn+Exoo` ziR9pOLE%JWYW4fJns1p4k|GV}7gzm|=z~!xMfI_{@f6d9`p#1ZK zCWZH~(_#7MSH8+Cr~LDVaBybKXpDfY#>E=okkP0BOMQ;vCj%oRA|$t|`!|;@)T#83 z^*KZ(+fFFVi;0a9iEG-{dN_-ZE<95`R-FXBszkf96cYg|MxAdnH8qKIS5TPgW>?X; zQM2zHWOvam=bUa+jbqA{8X5NaAo&^>8XfUL5wMkHBUq-t5nzFT(JA9B6+P)HxvBj< ziaTxGtl3xG)paWjb6vj|K$NRTQCOAMKFSkAC=Vr-XMM80Uq4l1b_w$NP0kqOUwb0T zha=mL-5j15YU2_-eLP{IA7}!I+)mX!ZQwX3Vw_xqWQ^WJpBtO6RU&4)oEV+)C|asH zLUvMPLH75^g7eiAZfYNfd2HZ=qsII9k^Udm{F-?3*8PCWsRXimhPL&?8bno+7A0_y zr8K3QU8E;ndjsmeFA>KQXJ`u(zsMHj^RIAnFfV$}PpQ@`1a~GDHnk7v#3ZG3nXu*X zC8y*6hFu_W7lTh=?vxt=%iecc#}g{D^6<9rNqk~uES(I;$r?|xEjQ>ER20lCA1|`O z{;dD!5CjW_|0F}UGNxe4C7_Br)_!_Ga#J-QvaJ&OmI_!`)#RHrk?SH9o-989oyl~I zmxBj&!vOf9q$$lo4n}FGX!jXK)*_XUo7(Ey3~f40jdnX>NEz$~d2pR{c{y}t96$h8 z&rp-$v+7u&x02l105{7GYuO3YO5z5@5nY3Hj@jN{r`*DW=91m?m+m6wX~^&|6H|eu z(P(CfWK*98YA7wNzAk7!X6+Opj_P8nsgE*-k95T?P)9GJ!o;7Bz!z3p`GXbDHvE1YU&*zHi-%10>N%2uK7bt2XB2FuLZHK(h+c`BD9L^7pBf)Z@ zBO{NS`*w`k^l4WZl*Q8lJ$LkJFn9x>c`n5txhYycxY< z?T%SNf6JQIII(rBNdbYEM#mXuupSv8zuBsHEes#-q+%O>Oup28>N#tt25DEB#Wf!F zEz#!cFTnt*)dD6aD!e565Hk%@N(x;7H-EDrLXsH^#H~WR>5MZcbjzm=hk*o6d7))) z!OGNm&(0M%Fe^aLjU15V$MerT+b=TWl!NsQi^dwv)<~n_WqB_9$Ob`DRr8Ml9FI;8 z4xWvC9QDetI_wgJ*8<`uxSeHZ%PmQn!8Ih`ULM5+O zMRrwF6!oI9TIN;C5X$x10UG?`0@)C|Bt${aU541mE87aG!IiC8O<*=INU1mdvPSY{ zc8F?hA17fNqA)U&&$EOWx!_&wHHEc8(dgj_$H5+XN#pmsv{|nmGlVg7O-0aV*Yc7! z0Dm{JWy;1<!RW9V(PQvRfi`XA0l-WRm-t9Tw{?E6KQU$+`}HJ-H&M4 zEM^R?*dpU89f7KNp3iBq?@P(!3%SiSV&rjw%0R~8dM(BeRibu#A)f3Y-kmn+*NZ)J_>K^9v>C z5bIOxatP<(21j1G6O!49kMTa=JNKKR&aTm>!fH*xq7hmgUHwd^=p=JvIhmk{51=Eo^u-R(LJ(Sm( zW0F6D0P2m#&`_EFs6GkA?;E>HrUlLAIP!W2xr*tRR02J~@f;9#mYUpSq_j0{L|7bn zvDLIfMNAO*j615nQYxUggq^Bj%=PNq^yS>4bs~jsEmtyaplpTuYDBvMz^ll#CF_kd zFo}?vqSfL&>3excEx_**N~_Lzu!7^>&Vs}cFekoK43>$XXX^^@{y1-4JutxFkJ9`% zPoK82be}unuIXQKrvvli_N*%@Z!m0K%Mqasb*q*I8V9~~8sgM?i<#xV%bMZq2b;$) z9zOY9Nu1gCS+<2GcfXdD4ZpBE_v?d{{FBB<8yG|lG!L2DaO`t?KOY~b^3u{vsC)Oe zw(eAI&>=Cw)zfq1mMyK)PoEx=lxGnb(OiE`xM^gQrcEOWt1h!z_@rhB-|FbjKa`*E zk^k^vF>Q1QjvPV9tq}v<8(IWMY>p``Yx(Kv*)^w6_X+v=ee(N$j*jg{9+~ip_%UPE z4?FZ9?&P!|Y1zamu~Vx^$Ez1E3`jq9>XPrt_6FLc`bS3iKhytn7&lN%@a@_?zKLOC zr)x#+J1CUduXH~>`#YR@(cZqk!{o9TJKNj08$Eh-M$m`0H*en5@8@{6sD0w8A$wzL zs!H9{&zyNhayx1A*fu#+;JykMRj*9Px$TIF6*>)oq7jHyjyr=S1q6PUc7jm zTyn1esVSbB!MbKFVyj zrliD|!g$*@ZQ3+3>`;4A-@&o!XiqbmfSadVkRG3*D(A_F}diyM~@-_s3(h26yl8y#i62Qm^kMZL#&FgbzmSy|?{da~Ab!_muIulHd5rQ*`_p zqF(j$$M3c~$6RQ>qt8YYEe@td+dt)xeiKtOvoKF{zakyoAL=}Io^kD$Hydp2>>7n% z38iIs#>|=OV^Ug-8Z~O%*d*J{Tehq#E}mKb&`}%g_{`_~cE)jh;nXWIc zs`67=(zP(*ujGb{WURUGkksPFF4a~$&w}G#wPF9TCLbQhW@J#@2YFxO+8wO!#B*y z$(f$7qkB!k!@B*YRew$5lMm~je7KH9vtH|I-gTi z2l4h@q1{exTJFWHw(XE;%ckux(&S~pq^fTr72iS7Lz32bn0t!s@YWLh61Rh7MBPx^-)kr>CQ>!MYcpqJJ(Z z`0~Y4Jv3|kMcpR#>)sw&T3TAlea>CAW%FkB!8(aAdgwLEd9(O{NB87?`vxcF4eAox z9_zpOtU;qN^~sYb^I08DwK=EkRuuYM^S9L|CN?(lYD2NpKQ+|pBNI zO$wYfBhbgrd$AAyqp-B--Pg>bk6CX^4+|?hD;vAswmmFt?JO)-tnb+V|NFxHg|lY+ zE|)L#F|)8X>)pqrw~d{3A3F=nUtj1kY{fmk;32=TB*@Qh=8PpiW=j{(R`|}I=VRBa zS1J;)2Fw$ ePrrV%YT27)&B?F)PVW` literal 0 HcmV?d00001 diff --git a/ui/plotters/__pycache__/plotter.cpython-310.pyc b/ui/plotters/__pycache__/plotter.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e2570040b6dbfea5536fc13f4e73f7f3553d0bc6 GIT binary patch literal 1248 zcmZuwJ#X7E5G5(emSx#>)2|Hu0~%^?1yU46fTHbE1TC^@AsCUi73jmIqy#m*#asVG zGUhLF?bNwjr`}Of;5sG1%j5B)_ulbbjz$52_4UVKal{DuiHF_gz~LBn^%{aCl4c~Q zF{OxE#&RciEOs(C_hOHdPei)X+Y;$rvA8FDrzGnCg8wBEHDhsB)mkKVuIgneA=CSq zE*h->c^_6~vP`gYce!vlhFyJzph!$5iJ7FznKQ|xvvp&)<%>N9Y_h*4S2XSe_T|80 zA8?=sP(PI6)`iixZlIButyuZAslRb4h+USKw^aL( zvtrwT_RIu{Q;~r@j?$_SvR1;Ft`9p2`$Z3Wh67`JHU+RD*93*YU%H_;Fg75LojccV zo3+^cS8C4$T2-pHB8-btTJgO^6c?g0uGHevg!8PamRwz^qOQOnT7oN4i+y9mW(Dci z)FbqSr9KN-$R>3Cq;s}Y^`O7lV>I;}$ML?aEzt;4dz2b!4zV5|#%1e+j%3H#C~33? z@m$%#$I5OY?kM$hIM_$DBzI0Vetn;H0=bu%e*s|~cAVJ3lW2&uRft@1ZUWBpQZ^al zkn?Yi$l4o?ucwC?aFR3%(a$iTBaQAgvg|P9plTc-?LolSRtFZ0sqZi+pv;HuJf-&j zBp8Z9zjsCsI<(=sE$|Co*cwzcyPn`WOkBq2QS Gf%6ySVglCy literal 0 HcmV?d00001 diff --git a/ui/plotters/gradients_plotter.py b/ui/plotters/gradients_plotter.py new file mode 100644 index 0000000..b675984 --- /dev/null +++ b/ui/plotters/gradients_plotter.py @@ -0,0 +1,7 @@ +from abc import ABC + +from matplotlib.figure import Figure + +from neural_net.epoch import Epoch +from neural_net.neural_net import NeuralNet +from ui.plotters.plotter import Plotter diff --git a/ui/plotters/loss_plotter.py b/ui/plotters/loss_plotter.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ui/plotters/loss_plotter.py @@ -0,0 +1 @@ + diff --git a/ui/plotters/plotter.py b/ui/plotters/plotter.py new file mode 100644 index 0000000..05ac16b --- /dev/null +++ b/ui/plotters/plotter.py @@ -0,0 +1,30 @@ +from abc import abstractmethod + +from matplotlib.figure import Figure + +from neural_net.epoch import Epoch + + +class Plotter: + def __init__(self, figure: Figure): + self.figure = figure + + def initialize_plots(self): + self.figure.show() + + @abstractmethod + def update_plot(self, data): + self.reset_plot() + + self.plot(data) + + self.figure.canvas.draw() + self.figure.canvas.flush_events() + + @abstractmethod + def reset_plot(self): + pass + + @abstractmethod + def plot(self, current_epoch: Epoch): + pass diff --git a/ui/plotters/predictions_plotter.py b/ui/plotters/predictions_plotter.py new file mode 100644 index 0000000..e69de29 diff --git a/ui/plotters/weights_plotter.py b/ui/plotters/weights_plotter.py new file mode 100644 index 0000000..e69de29 diff --git a/ui/training_page/__pycache__/training_page.cpython-310.pyc b/ui/training_page/__pycache__/training_page.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f352e87cec9c2d5c12a7dcb09f2bf2b7f33b31b5 GIT binary patch literal 3632 zcmZ`*-EP~+73Po>MbVNh%Zd|cx4X`wK+AU5&bCFdU9?5BizHh#Rf0xs7X%7|){JDz z6sZoWCNbnI8`vw~^tLEqU*;|P9CO<%KSCGiZoe~>WGT&(n3*$Y&itKk&QI28cm}=? z|5Y1)-8PJWQ)l(F(fJKZ{&!T|;4CqMvSopZzL{8KJFpq?tfVq_0!Pz!QXRX2tLaKo z8+(DLX(!nj*MqvIt4U+r44RsDlg)7}Xlc5ZY>nGNThm^$J?;b@W;{0d2CttRye`bq zturgw;q1BLH?DCmqw6!Z^(Y=rrT9Eb{}$!F;|Iebm`=$ARe3nc`bQvIT``qW(iKHd zMsX@6ciM<+ZpetbzHJF<{){_IV?)!K-lRoCOtL70$rqF0Y;2L6v)a zeWN%XK8uE;t7_3?66SQRKEb&7h_>b> z{}G77I5RHUj9oEi%*>e)nllTv#SPT<%($#v83e|2<2CF0j;iF-iIB<-!#ItLFqCyH zqB>a`7Nk+^9}dvuM}n)WWJ)xxl!M0p;D zuZ^!C9!z9568$1SNJJ!Q<%4*PyL}`@KO3g;iI4}=_+Yv11Y_&H{mCo2iKVwta(e1@ z=IFm}pWIqi*Z$%*{RLVOHi)jFe2$VAsAk3pnwT=_5p~H(GX-=uvPRGp>dK5khtS{1 zL8)?fQH2(19`%jTEovh@ceKIH3x=^e`1Q*TT4`qMwLh8JXLF5{)_tq%S5#x6f>1(_ zs+P)7(38%UcNi7@qcDd$)D}?C?64mt$zjw#4hWb99!oCXwNpfLC)6SU$6YvOA7=^eN-foldQ)))&zhye)qqg-A3OvJ!XG<(q4nJ+|s7Q z_35=QKBCD2G@>s-6mV)YbHx5({L?%m3ou6J$daF+wJ$ZASElAVD?P|_a|bT(*p#0_ zvbr*6jNG+e?WRX)E{bn=Pj3H_bO}}MEuQcm9eD3r!@zJl-&2)|jMG9@G^{07UYv(= znvy}N_ZCxOjY}64_OrCm-c8=aDyo*iUxgDmG}Rm~{9D=IPRPC84_}lHZk74*8q>El zZb5yyA&ww_faTvrjJEVq@(!vNbAedXM8V%?_J3^q{O z*+`q-8M!MmKYLUuoKY3U_MC)$`LOoQOcD)iTEpBzf_f2cXWf^vuK(N-&A%X zUKXm_&&FfS@R|GxmQ(g~c$}UjpideUQ1>29(>_Je*Gzs2Zctms;_D5P+MGw>J^g0) z?qAc~YZI0H;akQ(_TQ0o@G`=@PZYU2`5`JgaQ|4D#j&z_PoF9FrK&uV(O4+wvuROe zsj3&*aF~cPf~wYGf%sN(CUh-jKY945C*Q+bf!oarahPStnEbTYd-{82qy0is=#U@L zZdSy(tWtNMDwlYUzC|t%h;tv++%pD^NNXVv2Xvso)muY@B}INZXpt?4MRcewU1rIf zHi%@-_!&z6b5tF3hndXz*0bJ)-|Vm!N|RBqWwvzj%sNJ#Z(7!Z>R2Z`YgW1p;A^Hy z2#`gVzcaFqi`H;9qyl`-Xw&_NUbgp!lK5h&n(+UCjb+O<|uUc1>^q#$@GPda4< z9LUT#Gf&N#d1>mXZC+Yea6Uz4h^q7{uEOsLE%>=ttsifDx1sbg=!!G zhsAODBIcmv0~-E}s^6jVEoA{=<=}$IY;jke+rELcYR`euFQOFv3ufzV$1y$21=zNE z^8VVb|9_5b(3TBK>r%YBFjQU`BJNLN0->sh;a{gwQjSQ6j!0>h4$hL|sg5&}UY#VZ zt2!l~aepj|qm0WQu^RK-t}l`VS>p{>w`i7*^&pyFxxXxN{xunMPC&SZ%j$3mx-hi* H-?9D&YS*Ka literal 0 HcmV?d00001 diff --git a/ui/training_page/training_page.py b/ui/training_page/training_page.py new file mode 100644 index 0000000..a3d8135 --- /dev/null +++ b/ui/training_page/training_page.py @@ -0,0 +1,103 @@ +import threading +import tkinter as tk + +from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg +from matplotlib.figure import Figure + +from neural_net.epoch import Epoch +from neural_net.trainer import NeuralNetTrainer +from ui.app_state import AppState +from ui.front_page.plots.gradients import GradientsPlot +from ui.front_page.plots.layer_weights import LayerWeightsPlot +from ui.front_page.plots.loss import LossPlot +from ui.front_page.sections.training_information import EpochInformation + + +class TrainingPage(tk.Frame): + def __init__(self, parent, app_state: AppState, on_training_finished=None): + super().__init__(parent) + self.app_state = app_state + self.on_training_finished = on_training_finished + self.trainer: NeuralNetTrainer = None + # trainer = NeuralNetTrainer(self.app_state.neural_net, self.app_state.model_data, learning_rate, nr_epochs) + # self.app_state.trainers.append(trainer) + # self.trainer = trainer + self.create_ui() + + def start(self, learning_rate, nr_epochs, batch_size, callback=None): + if self.trainer is not None: + self.trainer.stop() + self.trainer = NeuralNetTrainer(self.app_state.neural_net, self.app_state.model_data, + learning_rate=learning_rate, nr_epochs=nr_epochs, batch_size=batch_size, + on_epoch_callback=self.update_training_data, + on_finished_callback=self.on_training_finished) + self.trainer.on_epoch_callback = self.update_training_data + self.thread = threading.Thread(target=self.trainer.start) + self.thread.start() + # self.trainer.start(on_epoch_finished=self.update_training_data) + if callback is not None: + callback() + + def update_training_data(self, training_run, data: Epoch): + print(f"Updating training data {data.epoch}") + if self.trainer.is_running: + self.training_information_container.update_training_data(training_run, data) + + self.loss_plot.update_training_data(training_run, data) + if data.epoch % 5 == 0: + self.gradients_plot.update_training_data(training_run, data) + self.layer0_weights_plot.update_training_data(training_run, data) + self.layer1_weights_plot.update_training_data(training_run, data) + + def create_ui(self): + # Training center + self.training_information_container = EpochInformation(self, self.app_state.neural_net, self.trainer) + self.training_information_container.pack(side=tk.TOP, fill=tk.X, expand=False, pady=10, padx=10, ipady=10, + ipadx=10) + + actions_frame = tk.Frame(self) + actions_frame.pack(side=tk.TOP, fill=tk.X, expand=False, pady=10, padx=10, ipady=10, ipadx=10) + btn_text = "Pause" + self.btn_toggle_pause = tk.Button(actions_frame, text=btn_text, command=self.toggle_state) + self.btn_toggle_pause.pack(side=tk.LEFT) + btn_stop = tk.Button(actions_frame, text="Stop", command=self.trainer.stop) + btn_stop.pack(side=tk.LEFT) + + # Plot tabs + plot_tab_control = tk.Notebook(self) + plot_tab_control.pack(side=tk.BOTTOM, fill=tk.BOTH, expand=True, pady=0, padx=0, ipady=0, ipadx=0) + + self.loss_plot = LossPlot(plot_tab_control, self.app_state.neural_net) + plot_tab_control.add(self.loss_plot, text="Loss Function") + + self.gradients_plot = GradientsPlot(plot_tab_control, self.app_state.neural_net) + plot_tab_control.add(self.gradients_plot, text="Gradients") + + self.layer0_weights_plot = LayerWeightsPlot(plot_tab_control, self.app_state.neural_net, + self.app_state.neural_net.layers[0], + 11, 11) + plot_tab_control.add(self.layer0_weights_plot, text="Weights layer 0") + + self.layer1_weights_plot = LayerWeightsPlot(plot_tab_control, self.app_state.neural_net, + self.app_state.neural_net.layers[1], + 2, 5) + plot_tab_control.add(self.layer1_weights_plot, text="Weights layer 1") + + def toggle_state(self): + self.trainer.toggle_state() + if self.trainer.training_paused: + self.btn_toggle_pause.config(text="Resume") + else: + self.btn_toggle_pause.config(text="Pause") + + @staticmethod + def create_plot_figure(tab): + figure = Figure() + + # Create a matplotlib canvas to display the plot + canvas = FigureCanvasTkAgg(figure, tab) + canvas.draw() + canvas.get_tk_widget().pack(fill=tk.BOTH, expand=True) + + return figure + diff --git a/ui/widgets.py b/ui/widgets.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/matplotlib/__pycache__/utils.cpython-310.pyc b/utils/matplotlib/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..92f21026c45378348ed26683248b0a68bf5544f5 GIT binary patch literal 666 zcmYjPv5wR*5S_7|-6iZohi-LWkOCr6Q2;_hqTGh=nv*Z~WpgC!V0*0$1aSA?=yB7p%qENAWya2_O zxQ7c=*=H!GN7@M$BX}@DWJ>;#+dtcn$|dP(ZD+)sc#UjSY`+wqxvCh>+LXUpAerwHvwKdA;!=ca)c|*Od=kUO8II zm)SKqJw&m70tt&g}`_}lj zxf&C%`r9zSHn!5ImT^osA(-JDU!#3aw2EUtSmYUb(&-J60d8QxJr1PQfwWRg@t+){ zeymP95aT2BVjz7^%7|6qYvY5ETW<;$MAgaO#=|YLFr`V|+I$OBtKC}mdemj-o9YYc z6*am|JWW_|`-Judp`@1~OB<3lBRq@Wo_2kEd8!(J(9H