TL;DR — Sick and tired of XML, jQuery, Base64, XHR, MIME, miltipart? Here is clean and simple WebSocket file transfer!
Background
When we started to craft N2O framework we strive to provide latest technologies on the market. Late in 2011 WebSocket protocol was not yet supported on all browsers and we spawned N2O on pure WebSocket stack blindly without knowing the future. Now we are sure it was right decision. We were sick and tired of XML, Base64, jQuery and XHR polling request known as comet. By numerous requests XHR is supported in N2O started from 2.4 but we engage everyone to decline this power-consuming technology.
Microservices vs Ad-hoc protocols
Once we got a project that involved Blu-ray disc file uploading through web browser. N2O was fit this case thanks to binary nature of WebSocket channel. We were pioneers who implemented simple and clean binary file upload, without all this multipart HTTP bullshit. We really engage anybody to deny both HTTP/1 and HTTP/2 profiles and use pure WebSocket channel. It is much easier, code is clean, no headers, no states, no codes, no mimes. You can build your own ad-hoc protocol easily without trying to fit your API language to resource-based REST endpoints architecture. Yes, we really think that all this microservices buzz are exactly just like design patterns: it solves nothing.
Naturally there still some cases REST is useful like existing systems integration; still WebSockets set of utilities lacks some usefull things like curl and wrk; but WebSockets is young technlology and tools like tcpkali arises and more to come.
Binary File Transfer Protocol is simple example of task that is hard to fit REST HTTP/2 approach. This is exactly where WebSocket protocol shines. FTP and BIN protocols were presented from the begining of formalizing N2O protocol stack and this document will guide you N2O FTP implementation. This implementation was made by the original N2O co-author Andrii Zadorozhnii and was designed to be limited by 100 LOC for both JavaScript and Erlang.
Erlang Supervision
Usually we encourage to build stateless N2O applications. This allows you easy scaling but sometimes you to really need access to global state, especialy in web frameworks: sessions, cache and other global resources like processes. Starting from N2O version 2.9 provides powerful n2o_async gen_server process under supervision. Currently N2O supports several clasess of async processes: file for file upload workers; async for web pages workers; and other custom clasess for you needs. The cool thing is that n2o_async hides verbose OTP gen_server templates and provides clean proc/2 callback API for any Erlang module. The other cool thing is that n2o_async supports gen_server's call, cast and info functions.
> pid(0,507,0) ! "hey".
Received: "hey"
ok
> n2o_async:send("ho!","hola").
Received: "hola"
ok
> gen_server:call(pid(0,507,0),"sync").
Received: "sync"
ok
> supervisor:which_children(n2o).
[{{async,
{"counter",<<"d43adcc79dd64393a1eb559227a2d3fd">>}},
<0.11564.0>,worker,
[n2o_async]}]
The only mandatory state value you need to hold during file uploading is a file offset that increments after each chunck. However user may want to put some additional metainformation from FTP packet during upload. For that purposes N2O implements File Transfer Protocol as supevised n2o_async backed by gen_server. By implementing N2O info/3 protocol API the FTP initialization is clean and portable:
info( #ftp { sid=Sid, filename=Filename, hash=Hash,
status= <<"init">>, offset=Size, block=B,
data=Msg } = FTP, Req, State) ->
application:set_env(n2o,formatter,bert),
Dir = lists:concat([?ROOT,'/',wf:to_list(Sid),'/']),
filelib:ensure_dir(Dir),
File = filename:join([Dir,Filename]),
FSize = case file:read_file_info(File) of
{ok, Fi} -> Fi#file_info.size;
{error, _} -> 0 end,
Name = { Sid, Filename, Hash },
Block = case B of 0 -> ?stop; _ -> ?next end,
Offset = case FSize >= Size of true -> FSize; false -> 0 end,
F2 = FTP#ftp{block = Block, offset = Offset, data = <<>> },
n2o_async:stop(file,Name),
n2o_async:start(#handler { module=?MODULE, class=file,
group=n2o, state=F2, name=Name }),
{reply,wf:format(F2),Req,State};
The async process needs only shifting state's offset and pushing it back to client
proc( #ftp{ sid=Sid, data=Msg, block=Block, filename=F} = FTP,
#handler{ state=#ftp{data=State, offset=Offset}} = Async) ->
F2 = FTP#ftp{status= "send", offset=Offset + Block },
wf:info(?MODULE,"send ~p", [F2#ftp{data= <<"">>}]),
case file:write_file(?FILE(Sid,F),<<Msg/binary>>,[append,raw]) of
ok -> {reply, F2#ftp{data=<<"">>}, Async#handler{state=F2}};
{error,Rw} -> {reply, {error, Rw}, Async} end.
JavaScript Client
The main idea was to create unified client for node.js and web browser. As you may want to use dropzone.js or other file uploader front ends. At the heart of ftp.js is a bert.js packaging as FTP is working only with BERT formatter. We deny using XHR or other transports for file uploading. However by abstracting enc client formatting function we are saving the ability to use any underlying transpots.
var ftp = {
$file: undefined,
$reader: undefined,
$block: undefined,
$offset: undefined,
init: function(file, force) {
ftp.$file = file; ftp.send('', 'init', 1); },
start: function() {
ftp.$active = true;
ftp.send_slice(ftp.$offset, ftp.$offset + ftp.$block); },
stop: function() { ftp.$active = false; },
send: function(data, status, force) {
ws.send(enc(tuple(atom('ftp'),number(1),bin(ftp.$file.name),
number(3),number(4),number(5),number(6),
number(7),bin(data),bin(status||'send'),
number(force || data.byteLength),number(11)))); },
send_slice: function(start, end) {
this.$reader = new FileReader();
this.$reader.onloadend=function(e) {
var res=e.target, data=e.target.result;
if (res.readyState == FileReader.DONE & data.byteLength < 0)
ftp.send(data); };
this.$reader.readAsArrayBuffer(ftp.$file.slice(start,end)); } }
$file.do = function(rsp) {
var offset = rsp.v[6].v, block = rsp.v[10].v, status = rsp.v[9].v;
switch (status) {
case 'init': ftp.$offset = offset; ftp.$block = block; break;
case 'send': var x = qi('ftp_status'); if(x) x.innerHTML = offset;
if (block>0 && ftp.$active)
ftp.send_slice(offset, offset+block); } }
N2O Element
The NITRO element will consist of three buttons: one select the file, other two start and stop uploading process. It should use only ftp.js API that can plugged to any upload front-end such as dropzone.js.
render_element(#upload{id=Id} = U) ->
Uid = case Id of undefined -> wf:temp_id(); I -> I end,
bind(ftp_open, click, "qi('upload').click(); e.codeventDefault();"),
bind(ftp_start, click, "ftp.start();"),
bind(ftp_stop, click, "ftp.stop();"),
bind(nitro:to_atom(Uid), change, "ftp.init(this.files[0],1);"),
Upload = #panel { body = [
#input { id = Uid, type = <<"file">> },
#span { id = ftp_status, body = [] },
#span { body = [
#button { id = ftp_open, body = "Browse" },
#button { id = ftp_start, body = "Upload" },
#button { id = ftp_stop, body = "Stop" }
] } ] }, wf:render(Upload).
bind(Control,Event,Code) ->
wf:wire(#bind{target=Control,type=Event,postback=Code}).