mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-21 00:00:13 +02:00
765943a4b7
in intranets and the internet can now choose to appear as Googlebot. This is an essential necessity to be able to compete in the field of commercial search appliances, since most web pages are these days optimized only for Google and no other search platform any more. All commercial search engine providers have a built-in fake-Google User Agent to be able to get the same search index as Google can do. Without the resistance against obeying to robots.txt in this case, no competition is possible any more. YaCy will always obey the robots.txt when it is used for crawling the web in a peer-to-peer network, but to establish a Search Appliance (like a Google Search Appliance, GSA) it is necessary to be able to behave exactly like a Google crawler. With this change, you will be able to switch the user agent when portal or intranet mode is selected on per-crawl-start basis. Every crawl start can have a different user agent.
108 lines
4.0 KiB
Java
108 lines
4.0 KiB
Java
import java.io.IOException;
|
|
import java.net.MalformedURLException;
|
|
import java.util.EnumMap;
|
|
import java.util.Iterator;
|
|
import java.util.regex.Pattern;
|
|
|
|
import net.yacy.cora.protocol.ClientIdentification;
|
|
import net.yacy.cora.protocol.RequestHeader;
|
|
import net.yacy.cora.util.ConcurrentLog;
|
|
import net.yacy.data.UserDB;
|
|
import net.yacy.data.ymark.YMarkAutoTagger;
|
|
import net.yacy.data.ymark.YMarkCrawlStart;
|
|
import net.yacy.data.ymark.YMarkEntry;
|
|
import net.yacy.data.ymark.YMarkMetadata;
|
|
import net.yacy.data.ymark.YMarkTables;
|
|
import net.yacy.data.ymark.YMarkUtil;
|
|
import net.yacy.document.Document;
|
|
import net.yacy.document.Parser.Failure;
|
|
import net.yacy.kelondro.data.meta.DigestURI;
|
|
import net.yacy.search.Switchboard;
|
|
import net.yacy.server.serverObjects;
|
|
import net.yacy.server.serverSwitch;
|
|
|
|
public class get_metadata {
|
|
|
|
static serverObjects prop;
|
|
|
|
public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
|
|
final Switchboard sb = (Switchboard) env;
|
|
prop = new serverObjects();
|
|
final UserDB.Entry user = sb.userDB.getUser(header);
|
|
final boolean isAdmin = (sb.verifyAuthentication(header));
|
|
final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT);
|
|
|
|
if(isAdmin || isAuthUser) {
|
|
|
|
final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN);
|
|
|
|
String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt());
|
|
boolean hasProtocol = false;
|
|
for (final YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) {
|
|
if(url.toLowerCase().startsWith(p.protocol())) {
|
|
hasProtocol = true;
|
|
break;
|
|
}
|
|
}
|
|
if (!hasProtocol) {
|
|
url=YMarkTables.PROTOCOLS.HTTP.protocol(url);
|
|
}
|
|
|
|
try {
|
|
final YMarkMetadata meta = new YMarkMetadata(new DigestURI(url), sb.index);
|
|
ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
|
|
final Document document = meta.loadDocument(sb.loader, agent);
|
|
final EnumMap<YMarkMetadata.METADATA, String> metadata = meta.loadMetadata();
|
|
|
|
prop.putXML("title", metadata.get(YMarkMetadata.METADATA.TITLE));
|
|
prop.putXML("desc", metadata.get(YMarkMetadata.METADATA.DESCRIPTION));
|
|
prop.put("keywords", putTags(document.dc_subject(','), "keywords"));
|
|
prop.put("autotags", putTags(YMarkAutoTagger.autoTag(document, 5, sb.tables.bookmarks.getTags(bmk_user)), "autotags"));
|
|
|
|
final YMarkCrawlStart crawlStart = new YMarkCrawlStart(sb.tables, url);
|
|
int count = 0;
|
|
if(!crawlStart.isEmpty()) {
|
|
final Iterator<String> iter = crawlStart.keySet().iterator();
|
|
String key;
|
|
while(iter.hasNext()) {
|
|
key = iter.next();
|
|
prop.putXML("crawlstart_"+count+"_key",key.toLowerCase());
|
|
prop.putXML("crawlstart_"+count+"_value",crawlStart.get(key));
|
|
count++;
|
|
}
|
|
}
|
|
prop.put("crawlstart", count);
|
|
|
|
} catch (final MalformedURLException e) {
|
|
// TODO Auto-generated catch block
|
|
ConcurrentLog.logException(e);
|
|
prop.put("status", "error");
|
|
} catch (final IOException e) {
|
|
// TODO Auto-generated catch block
|
|
ConcurrentLog.logException(e);
|
|
prop.put("status", "error");
|
|
} catch (final Failure e) {
|
|
// TODO Auto-generated catch block
|
|
ConcurrentLog.logException(e);
|
|
prop.put("status", "error");
|
|
}
|
|
} else {
|
|
prop.put(serverObjects.ACTION_AUTHENTICATE, YMarkTables.USER_AUTHENTICATE_MSG);
|
|
}
|
|
// return rewrite properties
|
|
return prop;
|
|
}
|
|
|
|
public static int putTags(final String tagString, final String var) {
|
|
final String list[] = Pattern.compile(YMarkUtil.TAGS_SEPARATOR).split(tagString, 0);
|
|
int count = 0;
|
|
for (final String element : list) {
|
|
final String tag = element;
|
|
if (!tag.equals("")) {
|
|
prop.putXML(var+"_"+count+"_tag", tag);
|
|
count++;
|
|
}
|
|
}
|
|
return count;
|
|
}
|
|
} |