Marown/yacy_search_server
1
0
Fork 0
mirror of https://github.com/yacy/yacy_search_server.git synced 2024-09-19 00:01:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

harmonization of http post of files for one and several files - this had

Browse Source 
been differently - and wrong for several files. also: base64-encoding
for gzipped push files because our data structures currently only
supports ASCII POST pushes..
This commit is contained in:
Michael Peter Christen 2016-03-11 08:59:33 +01:00
parent 1ce38fdaed
commit f12a900f3e
2 changed files with 24 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
htroot/api/push_p.java
View File

@ -74,7 +74,7 @@ public class push_p {
String collection = post.get("collection-" + i, "");
String lastModified = post.get("lastModified-" + i, ""); // must be in RFC1123 format
String contentType = post.get("contentType-" + i, "");
String data64 = post.get("data-" + i, ""); // file uploads are base64encoded in YaCyDefaultServlet.parseMultipart
String data64 = post.get("data-" + i + "$file", ""); // multi-file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
byte[] data = Base64Order.standardCoder.decode(data64);
if ((data == null || data.length == 0) && data64.length() > 0) data = UTF8.getBytes(data64); // for test cases

29
source/net/yacy/http/servlets/YaCyDefaultServlet.java
View File

@ -88,7 +88,6 @@ import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileItemFactory;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.eclipse.jetty.http.HttpHeader;
import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.MimeTypes;
@ -148,7 +147,7 @@ public class YaCyDefaultServlet extends HttpServlet {
protected ConcurrentHashMap<File, SoftReference<Method>> templateMethodCache = null;
// settings for multipart/form-data
protected static final File TMPDIR = new File(System.getProperty("java.io.tmpdir"));
protected static final int SIZE_FILE_THRESHOLD = 100 * 1024 * 1024; // 100 MB is a lot but appropriate for multi-document pushed using the push_p.json servlet
protected static final int SIZE_FILE_THRESHOLD = 1024 * 1024 * 1024; // 1GB is a lot but appropriate for multi-document pushed using the push_p.json servlet
protected static final FileItemFactory DISK_FILE_ITEM_FACTORY = new DiskFileItemFactory(SIZE_FILE_THRESHOLD, TMPDIR);
private final static TimeLimiter timeLimiter = new SimpleTimeLimiter(Executors.newCachedThreadPool());
/* ------------------------------------------------------------ */
@ -1114,8 +1113,19 @@ public class YaCyDefaultServlet extends HttpServlet {
}
}
if (files.size() <= 1) { // TODO: should include additonal checks to limit parameter.size below rel. large SIZE_FILE_THRESHOLD
for (Map.Entry<String, byte[]> fe: files) { // add the file content to parameter fieldname$file
args.put(fe.getKey()+"$file", (fe.getValue()));
for (Map.Entry<String, byte[]> job: files) { // add the file content to parameter fieldname$file
String n = job.getKey();
byte[] v = job.getValue();
String filename = args.get(n);
if (filename != null && filename.endsWith(".gz")) {
// transform this value into base64
String b64 = Base64Order.standardCoder.encode(v);
args.put(n + "$file", b64);
args.remove(n);
args.put(n, filename + ".base64");
} else {
args.put(n + "$file", v); // the byte[] is transformed into UTF8. You cannot push binaries here
}
}
} else {
// do this concurrently (this would all be superfluous if serverObjects could store byte[] instead only String)
@ -1129,8 +1139,15 @@ public class YaCyDefaultServlet extends HttpServlet {
public void run() {
Map.Entry<String, byte[]> job;
try {while ((job = files.take()) != POISON) {
String b64 = Base64Order.standardCoder.encode(job.getValue());
synchronized (args) {args.put(job.getKey(), b64);}
String n = job.getKey();
byte[] v = job.getValue();
String filename = args.get(n);
String b64 = Base64Order.standardCoder.encode(v);
synchronized (args) {
args.put(n + "$file", b64);
args.remove(n);
args.put(n, filename + ".base64");
}
}} catch (InterruptedException e) {}
}
};