Monday, June 6, 2011

Grab HTML from RSS Link

I'm getting very curious to save news from web. and i get suggestion from my friend, Sigit Adi Nugroho. He said we can get news content from rss. In rss we can get the link to news. then we grab news address and save to file .htm. I searching to many tutorial, finally i get the code from www.java2s.com . This code below will read rss and saving rss to file .txt then get link from there for grabbing and saving to html file.
I will waiting your comment for this code. Thanks u very much.

/**
 *
 * @author Zidny
 */
import com.sun.syndication.feed.synd.*;
import com.sun.syndication.feed.synd.SyndFeed;
import com.sun.syndication.io.SyndFeedInput;
import com.sun.syndication.io.XmlReader;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.SocketAddress;

import java.net.URL;
import java.net.URLConnection;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.SocketChannel;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.Iterator;

public class ParseRSSFeedUsingRome {

    //this code get from http://www.rgagnon.com/javadetails/java-0557.html
    public static void readRSS(String feed) throws Exception{
        String dir = "C:\\korpusberita\\";
        URL feedUrl = new URL(feed);

        SyndFeedInput input = new SyndFeedInput();
        SyndFeed sf = input.build(new XmlReader(feedUrl));

        List entries = sf.getEntries();
        Iterator it = entries.iterator();
        while (it.hasNext()) {
            SyndEntry entry = (SyndEntry)it.next();
            System.out.println(entry.getTitle());
            System.out.println(entry.getPublishedDate());
            System.out.println(entry.getLink());
            SyndContent description = entry.getDescription();
            System.out.println(description.getValue());
            System.out.println();

            File f;
            String fileName = dir+entry.getTitle()+".txt";
            try{
                // Create file
                FileWriter fstream = new FileWriter(fileName);
                BufferedWriter out = new BufferedWriter(fstream);
                SimpleDateFormat dateFormat = new SimpleDateFormat("ddMMyyyy");
                StringBuilder string = new StringBuilder(dateFormat.format(entry.getPublishedDate()));
                out.write("Title: "+entry.getTitle()+"\n");
                out.write("; Link: "+entry.getLink()+"\n");
                out.write("; Date: "+string+"\n");
                out.write("; Description: "+description.getValue()+"\n");
                //Close the output stream
                out.close();
                saveToHTML(entry.getTitle(), entry.getLink());
            }catch (Exception e){//Catch exception if any
                System.err.println("Error: " + e.getMessage());
            }
        }
    }

    //this code get from java2s.com
    public static void saveToHTML(String title, String link) throws IOException{
        URL u = new URL(link);
        String host = u.getHost();
        int port = 80;
        String file = "/";
        String fileName = "C:\\korpusberitahtml\\"+title+".htm";
        SocketAddress remote = new InetSocketAddress(host, port);
        SocketChannel channel = SocketChannel.open(remote);
        FileOutputStream out = new FileOutputStream(fileName);
        FileChannel localFile = out.getChannel();

        String request = "GET " + file + " HTTP/1.1\r\n" + "User-Agent: HTTPGrab\r\n"
            + "Accept: text/*\r\n" + "Connection: close\r\n" + "Host: " + host + "\r\n" + "\r\n";

        ByteBuffer header = ByteBuffer.wrap(request.getBytes("US-ASCII"));
        channel.write(header);

        ByteBuffer buffer = ByteBuffer.allocate(8192);
        while (channel.read(buffer) != -1) {
            buffer.flip();
            localFile.write(buffer);
            buffer.clear();
        }

        localFile.close();
        channel.close();
    }

    public static void main(String args[]) throws Exception {
        //String feed = "http://www.voanews.com/templates/Articles.rss?sectionPath=/indonesian/news";
        String[] feed = {"http://www.bbc.co.uk/indonesia/index.xml",
                         "http://www.voanews.com/templates/Articles.rss?sectionPath=/indonesian/news",
                         "http://detik.feedsportal.com/c/33613/f/589693/index.rss",
                         "http://republika.co.id/rss/koran",
                         "http://www.antara.co.id/rss/news.xml",
                         "http://news.okezone.com/rss",
                         "http://www.liputan6.com/feed/rss2/"};

        for(int i=0;i<feed.length;i++){
            readRSS(feed[i]);
        }
    }//end of main
}

No comments:

Post a Comment