diff --git a/group24/120509419/JUnitTest/download/FileDownloaderTest.java b/group24/120509419/JUnitTest/download/FileDownloaderTest.java new file mode 100644 index 0000000000..5d68c4d24e --- /dev/null +++ b/group24/120509419/JUnitTest/download/FileDownloaderTest.java @@ -0,0 +1,97 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package javaclass.download; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import static java.lang.System.in; +import java.math.BigInteger; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import javaclass.download.api.ConnectionManager; +import javaclass.download.api.DownloadListener; +import javaclass.download.impl.ConnectionManagerImpl; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * + * @author CJ + */ +public class FileDownloaderTest { + + boolean downloadFinished = false; + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testDownload() throws IOException, NoSuchAlgorithmException { + + String url = "http://pineappledb.xyz/test.txt"; + + FileDownloader downloader = new FileDownloader(url); + + ConnectionManager cm = new ConnectionManagerImpl(); + downloader.setConnectionManager(cm); + + downloader.setListener(new DownloadListener() { + @Override + public void notifyFinished() { + downloadFinished = true; + } + + }); + + downloader.setOutputFile(new File("C:\\Users\\CJ\\Desktop\\test000.txt")); + downloader.execute(); + + // 等待多线程下载程序执行完毕 + while (!downloadFinished) { + try { + System.out.println("还没有下载完成，休眠五秒"); + //休眠5秒 + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + System.out.println("下载完成！"); + + File downloadFile = downloader.getOutputFile(); + File expectedFile = new File("C:\\Users\\CJ\\Desktop\\test.txt"); + + long downloadSize = downloadFile.length(); + long expectedSize = expectedFile.length(); + + Assert.assertEquals(expectedSize, downloadSize); + FileInputStream downloadin = new FileInputStream(downloadFile); + FileInputStream expectedin = new FileInputStream(expectedFile); + + + MessageDigest md5 = MessageDigest.getInstance("MD5"); + md5.update(downloadin.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, downloadFile.length())); + BigInteger downloadbi = new BigInteger(1, md5.digest()); + md5.update(expectedin.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, expectedFile.length())); + BigInteger expectedbi = new BigInteger(1,md5.digest()); + Assert.assertEquals(downloadbi, expectedbi); + + } + +} diff --git a/group24/120509419/javaclass/download/DownloadThread.java b/group24/120509419/javaclass/download/DownloadThread.java new file mode 100644 index 0000000000..addc6a03c2 --- /dev/null +++ b/group24/120509419/javaclass/download/DownloadThread.java @@ -0,0 +1,70 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package javaclass.download; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; +import javaclass.download.api.Connection; + +public class DownloadThread extends Thread { + + Connection conn; + int startPos; + int endPos; + + int curStepByteSize = 1024*8; +// byte[] stepByteArr; + + public File getCurTmpFile() { + return curTmpFile; + } + File curTmpFile; + + public DownloadThread(Connection conn, int startPos, int endPos) { + +// stepByteArr = new byte[curStepByteSize]; + + this.conn = conn; + this.startPos = startPos; + this.endPos = endPos; +// curTmpFile = + } + + @Override + public void run() { + + try { + curTmpFile = File.createTempFile("TBtools", "downloadTmp"); +// BufferedWriter bw = new BufferedWriter(new FileWriter(curTmpFile)); + FileOutputStream fos = new FileOutputStream(curTmpFile); + + + // 一个线程 内 分批次下载，这样才不会挤爆内存 +// int size = endPos - startPos+1; +// int stepCount = size / curStepByteSize; +//// ssint remainSize = size % curStepByteSize; +// for (int i = 0; i < stepCount-1; i++) { +// fos.write(conn.read(startPos+curStepByteSize*i, startPos+curStepByteSize*(i+1)-1)); +// System.err.printf("Start Pos: %d\tEnd Pos %d\n",startPos+curStepByteSize*i, startPos+curStepByteSize*(i+1)-1); +// } + +// fos.write(conn.read(startPos+curStepByteSize*(stepCount-1),endPos)); +// System.err.printf("Start Pos: %d\tEnd Pos %d\n",startPos+curStepByteSize*(stepCount-1), endPos); + + fos.write(conn.read(startPos, endPos)); + + + fos.close(); + } catch (IOException ex) { + Logger.getLogger(DownloadThread.class.getName()).log(Level.SEVERE, null, ex); + } + } +} diff --git a/group24/120509419/javaclass/download/FileDownloader.java b/group24/120509419/javaclass/download/FileDownloader.java new file mode 100644 index 0000000000..b30826a72a --- /dev/null +++ b/group24/120509419/javaclass/download/FileDownloader.java @@ -0,0 +1,129 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package javaclass.download; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import javaclass.download.api.Connection; +import javaclass.download.api.ConnectionException; +import javaclass.download.api.ConnectionManager; +import javaclass.download.api.DownloadListener; + +public class FileDownloader { + + String url; + + DownloadListener listener; + + ConnectionManager cm; + + private int numOfThreads = 2; + + public File getOutputFile() { + return outputFile; + } + + public void setOutputFile(File outputFile) { + this.outputFile = outputFile; + } + + private File outputFile; + + + public FileDownloader(String _url) { + this.url = _url; + + + } + + public void execute() throws IOException { + // 在这里实现你的代码， 注意： 需要用多线程实现下载 + // 这个类依赖于其他几个接口, 你需要写这几个接口的实现代码 + // (1) ConnectionManager , 可以打开一个连接，通过Connection可以读取其中的一段（用startPos, endPos来指定） + // (2) DownloadListener, 由于是多线程下载， 调用这个类的客户端不知道什么时候结束，所以你需要实现当所有 + // 线程都执行完以后， 调用listener的notifiedFinished方法， 这样客户端就能收到通知。 + // 具体的实现思路： + // 1. 需要调用ConnectionManager的open方法打开连接， 然后通过Connection.getContentLength方法获得文件的长度 + // 2. 至少启动3个线程下载， 注意每个线程需要先调用ConnectionManager的open方法 + // 然后调用read方法， read方法中有读取文件的开始位置和结束位置的参数， 返回值是byte[]数组 + // 3. 把byte数组写入到文件中 + // 4. 所有的线程都下载完成以后， 需要调用listener的notifiedFinished方法 + + // 下面的代码是示例代码， 也就是说只有一个线程， 你需要改造成多线程的。 + Connection conn = null; + try { + // 针对每个线程，重新打开一个连接 + conn = cm.open(this.url); + + int length = conn.getContentLength(); + System.err.println("Total Length:"+length); + int threadSize = length / numOfThreads; +// int remainSize = length % numOfThreads; + int threadCount = 0; + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(numOfThreads); + DownloadThread[] dtArr = new DownloadThread[numOfThreads]; + for (int i = 0; i < numOfThreads-1; i++) { + conn = cm.open(this.url); + System.err.println("Thread "+threadCount++); + dtArr[i] = new DownloadThread(conn, i * threadSize, (i + 1) * threadSize - 1); + fixedThreadPool.submit(dtArr[i]); + } + conn = cm.open(this.url); + dtArr[numOfThreads-1] = new DownloadThread(conn,(numOfThreads-1) * threadSize, length - 1); + fixedThreadPool.submit(dtArr[numOfThreads-1]); + fixedThreadPool.shutdown(); + // 提交之后，等到 365天 + fixedThreadPool.awaitTermination(365, TimeUnit.DAYS); + + // 合并所有文件 + FileOutputStream fos = new FileOutputStream(outputFile); + for(DownloadThread curDt:dtArr){ + FileInputStream fis = new FileInputStream(curDt.getCurTmpFile()); + byte[] bufferedByteArr = new byte[1024*8]; + int readSize; + while((readSize=fis.read(bufferedByteArr))!=-1){ + fos.write(bufferedByteArr, 0, readSize); + } + fis.close(); + } + fos.close(); + + listener.notifyFinished(); + } catch (ConnectionException e) { + e.printStackTrace(); + } catch (InterruptedException ex) { + Logger.getLogger(FileDownloader.class.getName()).log(Level.SEVERE, null, ex); + } catch (FileNotFoundException ex) { + Logger.getLogger(FileDownloader.class.getName()).log(Level.SEVERE, null, ex); + } finally { + if (conn != null) { + conn.close(); + } + } + + } + + public void setListener(DownloadListener listener) { + this.listener = listener; + } + + public void setConnectionManager(ConnectionManager ucm) { + this.cm = ucm; + } + + public DownloadListener getListener() { + return this.listener; + } + +} diff --git a/group24/120509419/javaclass/download/api/Connection.java b/group24/120509419/javaclass/download/api/Connection.java new file mode 100644 index 0000000000..96d7df5d5e --- /dev/null +++ b/group24/120509419/javaclass/download/api/Connection.java @@ -0,0 +1,28 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package javaclass.download.api; + +import java.io.IOException; + +public interface Connection { + /** + * 给定开始和结束位置， 读取数据， 返回值是字节数组 + * @param startPos 开始位置， 从0开始 + * @param endPos 结束位置 + * @return + */ + public byte[] read(int startPos,int endPos) throws IOException; + /** + * 得到数据内容的长度 + * @return + */ + public int getContentLength(); + + /** + * 关闭连接 + */ + public void close(); +} \ No newline at end of file diff --git a/group24/120509419/javaclass/download/api/ConnectionException.java b/group24/120509419/javaclass/download/api/ConnectionException.java new file mode 100644 index 0000000000..5b8eb530dd --- /dev/null +++ b/group24/120509419/javaclass/download/api/ConnectionException.java @@ -0,0 +1,14 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package javaclass.download.api; + +/** + * + * @author CJ + */ +public class ConnectionException extends Exception { + +} \ No newline at end of file diff --git a/group24/120509419/javaclass/download/api/ConnectionManager.java b/group24/120509419/javaclass/download/api/ConnectionManager.java new file mode 100644 index 0000000000..8f0caeabd7 --- /dev/null +++ b/group24/120509419/javaclass/download/api/ConnectionManager.java @@ -0,0 +1,22 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package javaclass.download.api; + +/** + * + * @author CJ + */ +public interface ConnectionManager { + + /** + * 给定一个url , 打开一个连接 + * + * @param url + * @return + * @throws javaclass.download.api.ConnectionException + */ + public Connection open(String url) throws ConnectionException; +} diff --git a/group24/120509419/javaclass/download/api/DownloadListener.java b/group24/120509419/javaclass/download/api/DownloadListener.java new file mode 100644 index 0000000000..45d63dfa79 --- /dev/null +++ b/group24/120509419/javaclass/download/api/DownloadListener.java @@ -0,0 +1,14 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package javaclass.download.api; + +/** + * + * @author CJ + */ +public interface DownloadListener { + public void notifyFinished(); +} \ No newline at end of file diff --git a/group24/120509419/javaclass/download/impl/ConnectionImpl.java b/group24/120509419/javaclass/download/impl/ConnectionImpl.java new file mode 100644 index 0000000000..b14f2faaea --- /dev/null +++ b/group24/120509419/javaclass/download/impl/ConnectionImpl.java @@ -0,0 +1,60 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package javaclass.download.impl; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.net.URLConnection; +import javaclass.download.api.Connection; + +public class ConnectionImpl implements Connection { + + private URLConnection uc; + private int curPos; +// private ; + BufferedInputStream bis; + public ConnectionImpl(URLConnection uc) throws IOException{ + this.uc = uc; + this.curPos = 0; + bis = new BufferedInputStream(uc.getInputStream()); + } + + @Override + public byte[] read(int startPos, int endPos) throws IOException { + // 调整位置 +// bis.skip(startPos-this.curPos+1); +// bis.skip(startPos-this.curPos); +// this.curPos = endPos; + + // 因为只会read一次 + bis.skip(startPos); + + int readSize = endPos - startPos+1; + byte[] bufferedByte = new byte[readSize]; + + + int getSize = bis.read(bufferedByte); + System.err.println("Start Pos And End Pos:"+startPos+"\t"+endPos); + System.err.println("expected Size:"+readSize); + System.err.println("Get Size:"+getSize); + + return bufferedByte; +// = ; +// return Arrays.copyOfRange(bufferedByte, 0, getSize); + } + + @Override + public int getContentLength() { + + return uc.getContentLength(); + } + + @Override + public void close() { + + } + +} diff --git a/group24/120509419/javaclass/download/impl/ConnectionManagerImpl.java b/group24/120509419/javaclass/download/impl/ConnectionManagerImpl.java new file mode 100644 index 0000000000..4388fc17f7 --- /dev/null +++ b/group24/120509419/javaclass/download/impl/ConnectionManagerImpl.java @@ -0,0 +1,35 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package javaclass.download.impl; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.logging.Level; +import java.util.logging.Logger; +import javaclass.download.api.Connection; +import javaclass.download.api.ConnectionException; +import javaclass.download.api.ConnectionManager; + +public class ConnectionManagerImpl implements ConnectionManager { + + @Override + public Connection open(String url) throws ConnectionException { + + try { + URL curUrl = new URL(url); + ConnectionImpl ConnectionImpl = new ConnectionImpl(curUrl.openConnection()); + + return ConnectionImpl; + } catch (MalformedURLException ex) { + Logger.getLogger(ConnectionManagerImpl.class.getName()).log(Level.SEVERE, null, ex); + } catch (IOException ex) { + Logger.getLogger(ConnectionManagerImpl.class.getName()).log(Level.SEVERE, null, ex); + } + return null; + } + +} diff --git a/group24/120509419/javaclass/download/impl/Test.java b/group24/120509419/javaclass/download/impl/Test.java new file mode 100644 index 0000000000..b3a9d9f985 --- /dev/null +++ b/group24/120509419/javaclass/download/impl/Test.java @@ -0,0 +1,23 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package javaclass.download.impl; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; + +/** + * + * @author CJ + */ +public class Test { + public static void main(String[] args) throws FileNotFoundException, IOException { + File curFile = new File("http://www.baidu.com"); + RandomAccessFile raf = new RandomAccessFile(curFile,"r"); + System.err.println(raf.length()); + } +}