//bookmarksDB.java //------------------------------------- //part of YACY //(C) by Michael Peter Christen; mc@anomic.de //first published on http://www.anomic.de //Frankfurt, Germany, 2004 // //This file ist contributed by Alexander Schier // //This program is free software; you can redistribute it and/or modify //it under the terms of the GNU General Public License as published by //the Free Software Foundation; either version 2 of the License, or //(at your option) any later version. // //This program is distributed in the hope that it will be useful, //but WITHOUT ANY WARRANTY; without even the implied warranty of //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //GNU General Public License for more details. // //You should have received a copy of the GNU General Public License //along with this program; if not, write to the Free Software //Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // //Using this software in any meaning (reading, learning, copying, compiling, //running) means that you agree that the Author(s) is (are) not responsible //for cost, loss of data or any harm that may be caused directly or indirectly //by usage of this softare or this documentation. The usage of this software //is on your own risk. The installation and usage (starting/running) of this //software may allow other people or application to access your computer and //any attached devices and is highly dependent on the configuration of the //software which must be done by the user of the software; the author(s) is //(are) also not responsible for proper configuration and usage of the //software, even if provoked by documentation provided together with //the software. // //Any changes to this file according to the GPL as documented in the file //gpl.txt aside this file in the shipment you received can be done to the //lines that follows this copyright notice here, but changes must not be //done inside the copyright notive above. A re-distribution must contain //the intact and unchanged copyright notice. //Contributions and changes to the program code must be marked as such. package de.anomic.data; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.net.URL; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.TreeSet; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import org.w3c.dom.Document; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.htmlFilter.htmlFilterOutputStream; import de.anomic.kelondro.kelondroDyn; import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMap; import de.anomic.plasma.plasmaURL; import de.anomic.plasma.plasmaWordIndexEntry; import de.anomic.server.serverFileUtils; import de.anomic.server.logging.serverLog; public class bookmarksDB { kelondroMap tagsTable; kelondroMap bookmarksTable; kelondroMap datesTable; public static String tagHash(String tagName){ return plasmaWordIndexEntry.word2hash(tagName.toLowerCase()); } public static String dateToiso8601(Date date){ return new SimpleDateFormat("yyyy-MM-dd").format(date)+"T"+(new SimpleDateFormat("HH:mm:ss")).format(date)+"Z"; } public static Date iso8601ToDate(String iso8601){ String[] tmp=iso8601.split("T"); if(tmp.length!=2){ //Error parsing Date return new Date(); } String day=tmp[0]; String time=tmp[1]; if(time.length()>8){ time=time.substring(0,8); } try { Calendar date=Calendar.getInstance(); Calendar date2=Calendar.getInstance(); date.setTime(new SimpleDateFormat("yyyy-MM-dd").parse(day)); date2.setTime(new SimpleDateFormat("HH:mm:ss").parse(time)); date.set(Calendar.HOUR_OF_DAY, date2.get(Calendar.HOUR_OF_DAY)); date.set(Calendar.MINUTE, date2.get(Calendar.MINUTE)); date.set(Calendar.SECOND, date2.get(Calendar.SECOND)); return date.getTime(); } catch (ParseException e) { // TODO Auto-generated catch block e.printStackTrace(); } return new Date(); } public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, int bufferkb){ //bookmarks //check if database exists if(bookmarksFile.exists()){ try { //open it this.bookmarksTable=new kelondroMap(new kelondroDyn(bookmarksFile, 1024*bufferkb, '_')); } catch (IOException e) { //database reset :-(( bookmarksFile.delete(); bookmarksFile.getParentFile().mkdirs(); //urlHash is 12 bytes long this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, 12, 256, '_', true)); } }else{ //new database bookmarksFile.getParentFile().mkdirs(); this.bookmarksTable = new kelondroMap(new kelondroDyn(bookmarksFile, bufferkb * 1024, 12, 256, '_', true)); } //tags //check if database exists if(tagsFile.exists()){ try { //open it this.tagsTable=new kelondroMap(new kelondroDyn(tagsFile, 1024*bufferkb, '_')); } catch (IOException e) { //reset database tagsFile.delete(); tagsFile.getParentFile().mkdirs(); // max. 128 byte long tags this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, 12, 256, '_', true)); rebuildTags(); } }else{ //new database tagsFile.getParentFile().mkdirs(); this.tagsTable = new kelondroMap(new kelondroDyn(tagsFile, bufferkb * 1024, 12, 256, '_', true)); rebuildTags(); } // dates //check if database exists if(datesFile.exists()){ try { //open it this.datesTable=new kelondroMap(new kelondroDyn(datesFile, 1024*bufferkb, '_')); } catch (IOException e) { //reset database datesFile.delete(); datesFile.getParentFile().mkdirs(); //YYYY-MM-DDTHH:mm:ssZ = 20 byte. currently used: YYYY-MM-DD = 10 bytes this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, 20, 256, '_', true)); rebuildDates(); } }else{ //new database datesFile.getParentFile().mkdirs(); this.datesTable = new kelondroMap(new kelondroDyn(datesFile, bufferkb * 1024, 20, 256, '_', true)); rebuildDates(); } } public void close(){ try { bookmarksTable.close(); } catch (IOException e) {} try { tagsTable.close(); } catch (IOException e) {} try { datesTable.close(); } catch (IOException e) {} } public int bookmarksSize(){ return bookmarksTable.size(); } public int tagsSize(){ return tagsTable.size(); } /** * Store a Bookmark in the Bookmarkstable * @param bookmark the bookmark to store/update in the bookmarksTable */ public void setBookmarksTable(Bookmark bookmark){ try { bookmarksDB.this.bookmarksTable.set(bookmark.getUrlHash(), bookmark.mem); } catch (IOException e) {} } /** * store a Tag in the tagsDB or remove an empty tag * @param tag the tagobject to be stored/removed */ public void setTagsTable(Tag tag){ try { if(tag.size() >0){ bookmarksDB.this.tagsTable.set(tag.getTagHash(), tag.mem); }else{ bookmarksDB.this.tagsTable.remove(tag.getTagHash()); } } catch (IOException e) {} } public String addTag(Tag tag){ try { tagsTable.set(tag.getTagName(), tag.mem); return tag.getTagName(); } catch (IOException e) { return null; } } public void rebuildTags(){ serverLog.logInfo("BOOKMARKS", "rebuilding tags.db from bookmarks.db..."); Iterator it=bookmarkIterator(true); Bookmark bookmark; Tag tag; String[] tags; while(it.hasNext()){ bookmark=(Bookmark) it.next(); tags = bookmark.getTags().split(","); tag=null; for(int i=0;i0){ bookmarksDB.this.datesTable.set(getDateString(), mem); }else{ bookmarksDB.this.datesTable.remove(getDateString()); } } catch (IOException e) {} } public String getDateString(){ return date; } public int size(){ return listManager.string2arraylist(((String)this.mem.get(URL_HASHES))).size(); } } /** * Subclass, which stores the bookmark * */ public class Bookmark{ public static final String BOOKMARK_URL="bookmarkUrl"; public static final String BOOKMARK_TITLE="bookmarkTitle"; public static final String BOOKMARK_DESCRIPTION="bookmarkDesc"; public static final String BOOKMARK_TAGS="bookmarkTags"; public static final String BOOKMARK_PUBLIC="bookmarkPublic"; public static final String BOOKMARK_TIMESTAMP="bookmarkTimestamp"; private String urlHash; private Map mem; public Bookmark(String urlHash, Map map){ this.urlHash=urlHash; this.mem=map; } public Bookmark(String url){ if(!url.toLowerCase().startsWith("http://")){ url="http://"+url; } this.urlHash=plasmaURL.urlHash(url); mem=new HashMap(); mem.put(BOOKMARK_URL, url); try { Map oldmap= bookmarksTable.get(this.urlHash); if(oldmap != null && oldmap.containsKey(BOOKMARK_TIMESTAMP)){ mem.put(BOOKMARK_TIMESTAMP, oldmap.get(BOOKMARK_TIMESTAMP)); //preserve timestamp on edit }else{ mem.put(BOOKMARK_TIMESTAMP, String.valueOf(System.currentTimeMillis())); } bookmarksDate bmDate=getDate((String) mem.get(BOOKMARK_TIMESTAMP)); bmDate.add(this.urlHash); bmDate.setDatesTable(); removeBookmark(this.urlHash); //prevent empty tags } catch (IOException e) { //entry not yet present (normal case) mem.put(BOOKMARK_TIMESTAMP, String.valueOf(System.currentTimeMillis())); } } public Bookmark(String urlHash, URL url){ this.urlHash=urlHash; mem=new HashMap(); mem.put(BOOKMARK_URL, url.toString()); } public Bookmark(String urlHash, String url){ this.urlHash=urlHash; mem=new HashMap(); mem.put(BOOKMARK_URL, url); } public String getUrlHash(){ return urlHash; } public String getUrl(){ return (String) this.mem.get(BOOKMARK_URL); } public String getTags(){ if(this.mem.containsKey(BOOKMARK_TAGS)){ return (String)this.mem.get(BOOKMARK_TAGS); } return ""; } public ArrayList getTagsList(){ return listManager.string2arraylist(this.getTags()); } public String getDescription(){ if(this.mem.containsKey(BOOKMARK_DESCRIPTION)){ return (String) this.mem.get(BOOKMARK_DESCRIPTION); } return ""; } public String getTitle(){ if(this.mem.containsKey(BOOKMARK_TITLE)){ return (String) this.mem.get(BOOKMARK_TITLE); } return (String) this.mem.get(BOOKMARK_URL); } public boolean getPublic(){ if(this.mem.containsKey(BOOKMARK_PUBLIC)){ return ((String) this.mem.get(BOOKMARK_PUBLIC)).equals("public"); }else{ return false; } } public void setPublic(boolean isPublic){ if(isPublic){ this.mem.put(BOOKMARK_PUBLIC, "public"); }else{ this.mem.put(BOOKMARK_PUBLIC, "private"); } } public void setProperty(String name, String value){ mem.put(name, value); //setBookmarksTable(); } public void addTag(String tag){ ArrayList tags; if(!mem.containsKey(BOOKMARK_TAGS)){ tags=new ArrayList(); }else{ tags=listManager.string2arraylist((String) mem.get(BOOKMARK_TAGS)); } tags.add(tag); this.setTags(tags, true); } /** * set the Tags of the bookmark, and write them into the tags table. * @param tags a ArrayList with the tags */ public void setTags(ArrayList tags){ setTags(tags, true); } /** * set the Tags of the bookmark * @param tags ArrayList with the tagnames * @param local sets, whether the updated tags should be stored to tagsDB */ public void setTags(ArrayList tags, boolean local){ mem.put(BOOKMARK_TAGS, listManager.arraylist2string(tags)); Iterator it=tags.iterator(); while(it.hasNext()){ String tagName=(String) it.next(); Tag tag=getTag(tagHash(tagName)); if(tag == null){ tag=new Tag(tagName); } tag.add(getUrlHash()); if(local){ setTagsTable(tag); } } } public void setBookmarksTable(){ try { bookmarksDB.this.bookmarksTable.set(urlHash, mem); } catch (IOException e) {} } public long getTimeStamp(){ if(mem.containsKey(BOOKMARK_TIMESTAMP)){ return Long.parseLong((String)mem.get(BOOKMARK_TIMESTAMP)); }else{ return 0; } } public void setTimeStamp(long timestamp){ this.mem.put(BOOKMARK_TIMESTAMP, String.valueOf(timestamp)); } } public class tagIterator implements Iterator{ kelondroDyn.dynKeyIterator tagIter; bookmarksDB.Tag nextEntry; public tagIterator(boolean up) throws IOException { this.tagIter = bookmarksDB.this.tagsTable.keys(up, false); this.nextEntry = null; } public boolean hasNext() { try { return this.tagIter.hasNext(); } catch (kelondroException e) { //resetDatabase(); return false; } } public Object next() { try { return getTag((String) this.tagIter.next()); } catch (kelondroException e) { //resetDatabase(); return null; } } public void remove() { if (this.nextEntry != null) { try { Object tagName = this.nextEntry.getTagName(); if (tagName != null) removeTag((String) tagName); } catch (kelondroException e) { //resetDatabase(); } } } } public class bookmarkIterator implements Iterator{ kelondroDyn.dynKeyIterator bookmarkIter; bookmarksDB.Bookmark nextEntry; public bookmarkIterator(boolean up) throws IOException { this.bookmarkIter = bookmarksDB.this.bookmarksTable.keys(up, false); this.nextEntry = null; } public boolean hasNext() { try { return this.bookmarkIter.hasNext(); } catch (kelondroException e) { //resetDatabase(); return false; } } public Object next() { try { return getBookmark((String) this.bookmarkIter.next()); } catch (kelondroException e) { //resetDatabase(); return null; } } public void remove() { if (this.nextEntry != null) { try { Object bookmarkName = this.nextEntry.getUrlHash(); if (bookmarkName != null) removeBookmark((String) bookmarkName); } catch (kelondroException e) { //resetDatabase(); } } } } /** * Comparator to sort the Bookmarks with Timestamps */ public class bookmarkComparator implements Comparator{ private boolean newestFirst; /** * @param newestFirst newest first, or oldest first? */ public bookmarkComparator(boolean newestFirst){ this.newestFirst=newestFirst; } public int compare(Object obj1, Object obj2){ Bookmark bm1=getBookmark((String)obj1); Bookmark bm2=getBookmark((String)obj2); //XXX: what happens, if there is a big difference? (to much for int) if(this.newestFirst){ return (new Long(bm2.getTimeStamp() - bm1.getTimeStamp())).intValue(); }else{ return (new Long(bm1.getTimeStamp() - bm2.getTimeStamp())).intValue(); } } } /** * sorts the tag for name */ public class tagComparator implements Comparator{ public int compare(Object obj1, Object obj2){ return ((Tag)obj1).getTagName().compareTo(((Tag)obj2).getTagName()); } } }