vectorflow.serde source code

1 /**
2  * Internal serialization functions for automatic neural net ser/deser.
3  *
4  * Copyright: 2017 Netflix, Inc.
5  * License: $(LINK2 http://www.apache.org/licenses/LICENSE-2.0, Apache License Version 2.0)
6  */
7 module vectorflow.serde;
8 
9 private
10 {
11 import std.conv : to;
12 import std.stdio;
13 import std.traits;
14 
15 import vectorflow.neurallayer;
16 }
17 
18 /**
19 * Utility class used by vectorflow to serialize a net.
20 * @TODO: handle endianness properly for cross-arch compat.
21 */
22 class Serializer {
23 
24     File* _f;
25 
26     this(File* f)
27     {
28         _f = f;
29     }
30 
31     class EOFException : Exception{this(string msg){super(msg);}}
32 
33     void write(T)(T data) if(isSomeString!T || isBasicType!T || isUnsigned!T)
34     {
35         static if(isSomeString!T)
36         {
37             write_str(data.to!string);
38         }
39         else
40         {
41             _f.rawWrite((&data)[0..1]);
42         }
43     }
44 
45     private void write_str(string s)
46     {
47         write(s.length);
48         auto x = cast(ubyte[])s;
49         _f.rawWrite(x);
50     }
51 
52     final T read(T)() if(isSomeString!T || isBasicType!T || isUnsigned!T)
53     {
54         scope(failure){ if(_f.eof) throw new EOFException("");}
55         scope(exit){ if(_f.eof) throw new EOFException("");}
56         static if(isSomeString!T)
57         {
58             ulong str_sz;
59             _f.rawRead((&str_sz)[0..1]);
60             auto str = new ubyte[str_sz];
61             str = _f.rawRead(str);
62             return cast(string)str;
63         }
64         else
65         {
66             T d;
67             _f.rawRead((&d)[0..1]);
68             return d;
69         }
70     }
71 
72     void write_vec(T)(T vec) if(isArray!T)
73     {
74         write(vec.length);
75         foreach(v; vec)
76             write(v);
77     }
78 
79     T[] read_vec(T)()
80     {
81         auto len = read!ulong();
82         auto res = new T[len];
83         foreach(i; 0..len)
84             res[i] = read!T();
85 
86         return res;
87     }
88 
89     NeuralLayer[] deserialize_layers()
90     {
91         NeuralLayer[] layers;
92 
93         while(!_f.eof)
94         {
95             string layer_type;
96             try{ layer_type = read!string(); }
97             catch(EOFException e){ break; } 
98 
99             auto l = cast(NeuralLayer)Object.factory(layer_type);
100             l.deser(this);
101             layers ~= l;
102             writeln("Deserialized ", l.to!string);
103         }
104 
105         return layers;
106     }
107 }