mirror of
https://github.com/yacy/yacy_search_server.git
synced 2024-09-19 00:01:41 +02:00
using guava for host resolution (non-blocking for ips) and time-out
This commit is contained in:
Michael Peter Christen
parent
f094936b89
commit
90b82ce994
|
|
@ -48,6 +48,9 @@
|
|||
<classpathentry kind="lib" path="lib/apache-solr-solrj-3.6.0.jar" sourcepath="/Users/admin/git/lucene-solr/solr/solrj/src/java"/>
|
||||
<classpathentry kind="lib" path="lib/commons-compress-1.4.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/apache-solr-core-3.6.0.jar" sourcepath="/Users/admin/git/lucene-solr/solr/core/src/java/"/>
|
||||
<classpathentry kind="lib" path="lib/guava-r05.jar"/>
|
||||
<classpathentry kind="lib" path="lib/log4j-over-slf4j-1.6.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/commons-lang-2.6.jar"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
|
||||
<classpathentry kind="lib" path="lib/icu4j-core.jar"/>
|
||||
<classpathentry kind="lib" path="lib/htmllexer.jar"/>
|
||||
|
|
|
|||
|
|
@ -43,7 +43,10 @@ import java.util.List;
|
|||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
|
|
@ -54,6 +57,10 @@ import net.yacy.cora.storage.KeyList;
|
|||
import net.yacy.kelondro.logging.Log;
|
||||
import net.yacy.kelondro.util.MemoryControl;
|
||||
|
||||
import com.google.common.net.InetAddresses;
|
||||
import com.google.common.util.concurrent.SimpleTimeLimiter;
|
||||
import com.google.common.util.concurrent.TimeLimiter;
|
||||
|
||||
public class Domains {
|
||||
|
||||
|
||||
|
|
@ -553,14 +560,16 @@ public class Domains {
|
|||
cacheHit_Insert++;
|
||||
}
|
||||
|
||||
final private static TimeLimiter timeLimiter = new SimpleTimeLimiter(Executors.newFixedThreadPool(20));
|
||||
|
||||
/**
|
||||
* resolve a host address using a local DNS cache and a DNS lookup if necessary
|
||||
* @param host
|
||||
* @return the hosts InetAddress or null if the address cannot be resolved
|
||||
*/
|
||||
public static InetAddress dnsResolve(String host) {
|
||||
if ((host == null) || (host.length() == 0)) return null;
|
||||
host = host.toLowerCase().trim();
|
||||
public static InetAddress dnsResolve(final String host0) {
|
||||
if ((host0 == null) || (host0.length() == 0)) return null;
|
||||
final String host = host0.toLowerCase().trim();
|
||||
// try to simply parse the address
|
||||
InetAddress ip = parseInetAddress(host);
|
||||
if (ip != null) return ip;
|
||||
|
|
@ -615,8 +624,23 @@ public class Domains {
|
|||
try {
|
||||
//final long t = System.currentTimeMillis();
|
||||
Thread.currentThread().setName("Domains: DNS resolve of '" + host + "'"); // thread dump show which host is resolved
|
||||
ip = TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone
|
||||
//ip = InetAddress.getByName(host); // this makes the DNS request to backbone
|
||||
if (InetAddresses.isInetAddress(host)) {
|
||||
try {
|
||||
ip = InetAddresses.forString(host);
|
||||
Log.logInfo("Domains", "using guava for host resolution:" + host);
|
||||
} catch (IllegalArgumentException e) {
|
||||
ip = null;
|
||||
}
|
||||
}
|
||||
if (ip == null) {
|
||||
ip = timeLimiter.callWithTimeout(new Callable<InetAddress>() {
|
||||
@Override
|
||||
public InetAddress call() throws Exception {
|
||||
return InetAddress.getByName(host);
|
||||
}
|
||||
}, 1000L, TimeUnit.MILLISECONDS, false);
|
||||
//ip = TimeoutRequest.getByName(host, 1000); // this makes the DNS request to backbone
|
||||
}
|
||||
//.out.println("DNSLOOKUP-*LOOKUP* " + host + ", time = " + (System.currentTimeMillis() - t) + "ms");
|
||||
} catch (final Throwable e) {
|
||||
// add new entries
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ public class TimeoutRequest<E> {
|
|||
try {
|
||||
final Future<E> taskFuture = service.submit(this.call);
|
||||
final Runnable t = new Runnable() {
|
||||
@Override
|
||||
public void run() { taskFuture.cancel(true); }
|
||||
};
|
||||
service.execute(t);
|
||||
|
|
@ -109,6 +110,7 @@ public class TimeoutRequest<E> {
|
|||
*/
|
||||
public static boolean ping(final String host, final int port, final int timeout) throws ExecutionException {
|
||||
return new TimeoutRequest<Boolean>(new Callable<Boolean>() {
|
||||
@Override
|
||||
public Boolean call() {
|
||||
//long time = System.currentTimeMillis();
|
||||
try {
|
||||
|
|
@ -133,25 +135,6 @@ public class TimeoutRequest<E> {
|
|||
}).call(timeout).booleanValue();
|
||||
}
|
||||
|
||||
/**
|
||||
* do a DNS lookup within a given time
|
||||
* @param host
|
||||
* @param timeout
|
||||
* @return the InetAddress for a given domain name
|
||||
* @throws ExecutionException
|
||||
*/
|
||||
public static InetAddress getByName(final String host, final long timeout) throws ExecutionException {
|
||||
return new TimeoutRequest<InetAddress>(new Callable<InetAddress>() {
|
||||
public InetAddress call() {
|
||||
try {
|
||||
return InetAddress.getByName(host);
|
||||
} catch (final UnknownHostException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}).call(timeout);
|
||||
}
|
||||
|
||||
/**
|
||||
* perform a reverse domain name lookup for a given InetAddress within a given timeout
|
||||
* @param i
|
||||
|
|
@ -161,6 +144,7 @@ public class TimeoutRequest<E> {
|
|||
*/
|
||||
public static String getHostName(final InetAddress i, final long timeout) throws ExecutionException {
|
||||
return new TimeoutRequest<String>(new Callable<String>() {
|
||||
@Override
|
||||
public String call() { return i.getHostName(); }
|
||||
}).call(timeout);
|
||||
}
|
||||
|
|
@ -175,6 +159,7 @@ public class TimeoutRequest<E> {
|
|||
public static boolean exists(final SmbFile file, final long timeout) throws IOException {
|
||||
try {
|
||||
return new TimeoutRequest<Boolean>(new Callable<Boolean>() {
|
||||
@Override
|
||||
public Boolean call() { try {
|
||||
return file.exists();
|
||||
} catch (final SmbException e) {
|
||||
|
|
@ -196,6 +181,7 @@ public class TimeoutRequest<E> {
|
|||
public static boolean canRead(final SmbFile file, final long timeout) throws IOException {
|
||||
try {
|
||||
return new TimeoutRequest<Boolean>(new Callable<Boolean>() {
|
||||
@Override
|
||||
public Boolean call() { try {
|
||||
return file.canRead();
|
||||
} catch (final SmbException e) {
|
||||
|
|
@ -217,6 +203,7 @@ public class TimeoutRequest<E> {
|
|||
public static boolean canWrite(final SmbFile file, final long timeout) throws IOException {
|
||||
try {
|
||||
return new TimeoutRequest<Boolean>(new Callable<Boolean>() {
|
||||
@Override
|
||||
public Boolean call() { try {
|
||||
return file.canWrite();
|
||||
} catch (final SmbException e) {
|
||||
|
|
@ -238,6 +225,7 @@ public class TimeoutRequest<E> {
|
|||
public static boolean isHidden(final SmbFile file, final long timeout) throws IOException {
|
||||
try {
|
||||
return new TimeoutRequest<Boolean>(new Callable<Boolean>() {
|
||||
@Override
|
||||
public Boolean call() { try {
|
||||
return file.isHidden();
|
||||
} catch (final SmbException e) {
|
||||
|
|
@ -259,6 +247,7 @@ public class TimeoutRequest<E> {
|
|||
public static boolean isDirectory(final SmbFile file, final long timeout) throws IOException {
|
||||
try {
|
||||
return new TimeoutRequest<Boolean>(new Callable<Boolean>() {
|
||||
@Override
|
||||
public Boolean call() { try {
|
||||
return file.isDirectory();
|
||||
} catch (final SmbException e) {
|
||||
|
|
@ -280,6 +269,7 @@ public class TimeoutRequest<E> {
|
|||
public static long length(final SmbFile file, final long timeout) throws IOException {
|
||||
try {
|
||||
return new TimeoutRequest<Long>(new Callable<Long>() {
|
||||
@Override
|
||||
public Long call() { try {
|
||||
return file.length();
|
||||
} catch (final SmbException e) {
|
||||
|
|
@ -301,6 +291,7 @@ public class TimeoutRequest<E> {
|
|||
public static long lastModified(final SmbFile file, final long timeout) throws IOException {
|
||||
try {
|
||||
return new TimeoutRequest<Long>(new Callable<Long>() {
|
||||
@Override
|
||||
public Long call() { try {
|
||||
return file.lastModified();
|
||||
} catch (final SmbException e) {
|
||||
|
|
@ -322,6 +313,7 @@ public class TimeoutRequest<E> {
|
|||
public static String[] list(final SmbFile file, final long timeout) throws IOException {
|
||||
try {
|
||||
return new TimeoutRequest<String[]>(new Callable<String[]>() {
|
||||
@Override
|
||||
public String[] call() { try {
|
||||
return file.list();
|
||||
} catch (final SmbException e) {
|
||||
|
|
@ -334,11 +326,4 @@ public class TimeoutRequest<E> {
|
|||
}
|
||||
}
|
||||
|
||||
public static void main(final String[] args) {
|
||||
try {
|
||||
System.out.println(getByName("yacy.net", 100));
|
||||
} catch (final ExecutionException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user