Major https://foo.com/path のような URL から robots.txt の URL を作成すると
https:/robots.txt となってしまいます．
Index: src/main/java/org/seasar/robot/client/http/CommonsHttpClient.java
===================================================================
--- src/main/java/org/seasar/robot/client/http/CommonsHttpClient.java (revision 121)
+++ src/main/java/org/seasar/robot/client/http/CommonsHttpClient.java (working copy)
@@ -145,7 +145,7 @@
return;
}
- int idx = url.indexOf('/', 7);
+ int idx = url.indexOf('/', url.indexOf("://") + 3);
String hostUrl;
if (idx >= 0) {
hostUrl = url.substring(0, idx);