1 // HTTP utils based on https://github.com/adamdruppe/arsd/blob/master/http.d 2 // copyright Adam D. Ruppe 3 module HttpClient; 4 5 import std.socket; 6 import std..string; 7 import std.conv; 8 static import std.algorithm; 9 static import std.uri; 10 11 ubyte[] getBinary(string url, string[string] cookies = null) { 12 auto hr = httpRequest("GET", url, null, cookies); 13 14 const uint maxAttempts = 32; 15 uint attempts = 0; 16 17 while (hr.code == 302 && attempts < maxAttempts) { 18 hr = httpRequest("GET", hr.headers["Location"], null, cookies); 19 ++attempts; 20 } 21 22 if(hr.code != 200) 23 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 24 25 return hr.content; 26 } 27 28 struct HttpResponse { 29 int code; 30 string contentType; 31 string[string] cookies; 32 string[string] headers; 33 ubyte[] content; 34 } 35 36 struct UriParts { 37 string original; 38 string method; 39 string host; 40 ushort port; 41 string path; 42 43 bool useHttps; 44 45 this(string uri) { 46 original = uri; 47 48 if(uri[0 .. 8] == "https://") 49 useHttps = true; 50 else 51 if(uri[0..7] != "http://") 52 throw new Exception("You must use an absolute, http or https URL."); 53 54 version(with_openssl) {} else 55 if(useHttps) 56 throw new Exception("openssl support not compiled in try -version=with_openssl"); 57 58 int start = useHttps ? 8 : 7; 59 60 auto posSlash = uri[start..$].indexOf("/"); 61 if(posSlash != -1) 62 posSlash += start; 63 64 if(posSlash == -1) 65 posSlash = uri.length; 66 67 auto posColon = uri[start..$].indexOf(":"); 68 if(posColon != -1) 69 posColon += start; 70 71 if(useHttps) 72 port = 443; 73 else 74 port = 80; 75 76 if(posColon != -1 && posColon < posSlash) { 77 host = uri[start..posColon]; 78 port = to!ushort(uri[posColon+1..posSlash]); 79 } else 80 host = uri[start..posSlash]; 81 82 path = uri[posSlash..$]; 83 if(path == "") 84 path = "/"; 85 } 86 } 87 88 HttpResponse httpRequest(string method, string uri, const(ubyte)[] content = null, string[string] cookies = null, string[] headers = null) { 89 import std.socket; 90 91 auto u = UriParts(uri); 92 // auto f = openNetwork(u.host, u.port); 93 auto f = new TcpSocket(); 94 f.connect(new InternetAddress(u.host, u.port)); 95 96 void delegate(string) write = (string d) { 97 f.send(d); 98 }; 99 100 char[4096] readBuffer; // rawRead actually blocks until it can fill up the whole buffer... which is broken as far as http goes so one char at a time i guess. slow lol 101 char[] delegate() read = () { 102 size_t num = f.receive(readBuffer); 103 return readBuffer[0..num]; 104 }; 105 106 version(with_openssl) { 107 import deimos.openssl.ssl; 108 SSL* ssl; 109 SSL_CTX* ctx; 110 if(u.useHttps) { 111 void sslAssert(bool ret){ 112 if (!ret){ 113 throw new Exception("SSL_ERROR"); 114 } 115 } 116 SSL_library_init(); 117 OpenSSL_add_all_algorithms(); 118 SSL_load_error_strings(); 119 120 ctx = SSL_CTX_new(SSLv3_client_method()); 121 sslAssert(!(ctx is null)); 122 123 ssl = SSL_new(ctx); 124 SSL_set_fd(ssl, f.handle); 125 sslAssert(SSL_connect(ssl) != -1); 126 127 write = (string d) { 128 SSL_write(ssl, d.ptr, cast(uint)d.length); 129 }; 130 131 read = () { 132 auto len = SSL_read(ssl, readBuffer.ptr, readBuffer.length); 133 return readBuffer[0 .. len]; 134 }; 135 } 136 } 137 138 139 HttpResponse response = doHttpRequestOnHelpers(write, read, method, uri, content, cookies, headers, u.useHttps); 140 141 version(with_openssl) { 142 if(u.useHttps) { 143 SSL_free(ssl); 144 SSL_CTX_free(ctx); 145 } 146 } 147 148 return response; 149 } 150 151 /** 152 Executes a generic http request, returning the full result. The correct formatting 153 of the parameters are the caller's responsibility. Content-Length is added automatically, 154 but YOU must give Content-Type! 155 */ 156 HttpResponse doHttpRequestOnHelpers(void delegate(string) write, char[] delegate() read, string method, string uri, const(ubyte)[] content = null, string[string] cookies = null, string[] headers = null, bool https = false) 157 in { 158 assert(method == "POST" || method == "GET"); 159 } 160 body { 161 auto u = UriParts(uri); 162 163 164 165 166 167 write(format("%s %s HTTP/1.1\r\n", method, u.path)); 168 write(format("Host: %s\r\n", u.host)); 169 write(format("Connection: close\r\n")); 170 if(content !is null) 171 write(format("Content-Length: %d\r\n", content.length)); 172 173 if(cookies !is null) { 174 string cookieHeader = "Cookie: "; 175 bool first = true; 176 foreach(k, v; cookies) { 177 if(first) 178 first = false; 179 else 180 cookieHeader ~= "; "; 181 cookieHeader ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v); 182 } 183 184 write(format("%s\r\n", cookieHeader)); 185 } 186 187 if(headers !is null) 188 foreach(header; headers) 189 write(format("%s\r\n", header)); 190 write("\r\n"); 191 if(content !is null) 192 write(cast(string) content); 193 194 195 string buffer; 196 197 string readln() { 198 auto idx = buffer.indexOf("\r\n"); 199 if(idx == -1) { 200 auto more = read(); 201 if(more.length == 0) { // end of file or something 202 auto ret = buffer; 203 buffer = null; 204 return ret; 205 } 206 buffer ~= more; 207 return readln(); 208 } 209 auto ret = buffer[0 .. idx + 2]; // + the \r\n 210 if(idx + 2 < buffer.length) 211 buffer = buffer[idx + 2 .. $]; 212 else 213 buffer = null; 214 return ret; 215 } 216 217 HttpResponse hr; 218 cont: 219 string l = readln(); 220 if(l[0..9] != "HTTP/1.1 ") 221 throw new Exception("Not talking to a http server"); 222 223 hr.code = to!int(l[9..12]); // HTTP/1.1 ### OK 224 225 if(hr.code == 100) { // continue 226 do { 227 l = readln(); 228 } while(l.length > 1); 229 230 goto cont; 231 } 232 233 bool chunked = false; 234 235 auto line = readln(); 236 while(line.length) { 237 if(line.strip.length == 0) 238 break; 239 240 import std.algorithm : findSplit; 241 import std.range : empty; 242 auto split = findSplit(line, ":"); 243 244 if (!split[0].empty && !split[1].empty) 245 hr.headers[split[0]] = split[2].strip(); 246 247 if(line.startsWith("Content-Type: ")) 248 hr.contentType = line[14..$-1]; 249 if(line.startsWith("Set-Cookie: ")) { 250 auto hdr = line["Set-Cookie: ".length .. $-1]; 251 auto semi = hdr.indexOf(";"); 252 if(semi != -1) 253 hdr = hdr[0 .. semi]; 254 255 auto equal = hdr.indexOf("="); 256 string name, value; 257 if(equal == -1) { 258 name = hdr; 259 // doesn't this mean erase the cookie? 260 } else { 261 name = hdr[0 .. equal]; 262 value = hdr[equal + 1 .. $]; 263 } 264 265 name = std.uri.decodeComponent(name); 266 value = std.uri.decodeComponent(value); 267 268 hr.cookies[name] = value; 269 } 270 if(line.startsWith("Transfer-Encoding: chunked")) 271 chunked = true; 272 line = readln(); 273 } 274 275 // there might be leftover stuff in the line buffer 276 ubyte[] response = cast(ubyte[]) buffer.dup; 277 auto part = read(); 278 while(part.length) { 279 response ~= part; 280 part = read(); 281 } 282 283 if(chunked) { 284 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 285 // read binary data of that length. it is our content 286 // repeat until a zero sized chunk 287 // then read footers as headers. 288 289 int state = 0; 290 int size; 291 int start = 0; 292 for(int a = 0; a < response.length; a++) { 293 final switch(state) { 294 case 0: // reading hex 295 char c = response[a]; 296 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 297 // just keep reading 298 } else { 299 int power = 1; 300 size = 0; 301 for(int b = a-1; b >= start; b--) { 302 char cc = response[b]; 303 if(cc >= 'a' && cc <= 'z') 304 cc -= 0x20; 305 int val = 0; 306 if(cc >= '0' && cc <= '9') 307 val = cc - '0'; 308 else 309 val = cc - 'A' + 10; 310 311 size += power * val; 312 power *= 16; 313 } 314 state++; 315 continue; 316 } 317 break; 318 case 1: // reading until end of line 319 char c = response[a]; 320 if(c == '\n') { 321 if(size == 0) 322 state = 3; 323 else 324 state = 2; 325 } 326 break; 327 case 2: // reading data 328 hr.content ~= response[a..a+size]; 329 a += size; 330 a+= 1; // skipping a 13 10 331 start = a + 1; 332 state = 0; 333 break; 334 case 3: // reading footers 335 goto done; // FIXME 336 } 337 } 338 } else 339 hr.content = response; 340 done: 341 342 return hr; 343 }