diff --git a/group24/494800949/build.gradle b/group24/494800949/build.gradle index 7930ebfca5..dd69528893 100644 --- a/group24/494800949/build.gradle +++ b/group24/494800949/build.gradle @@ -20,6 +20,8 @@ dependencies{ compile group: 'junit', name: 'junit', version: '4.11' compile files("lib/dom4j-1.6.1.jar") compile files("lib/jaxen-1.1.1.jar") + compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.2' + } gradle.projectsEvaluated { tasks.withType(JavaCompile) { diff --git a/group24/494800949/src/main/java/com/coding/week3/download/DownloadThread.java b/group24/494800949/src/main/java/com/coding/week3/download/DownloadThread.java new file mode 100644 index 0000000000..a2a51cd583 --- /dev/null +++ b/group24/494800949/src/main/java/com/coding/week3/download/DownloadThread.java @@ -0,0 +1,52 @@ +package com.coding.week3.download; + + +import com.coding.week3.download.api.Connection; + +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.util.concurrent.atomic.AtomicInteger; + +public class DownloadThread extends Thread { + private static final int BUFF_SIZE = 1024; + Connection conn; + int startPos; + int endPos; + RandomAccessFile ras; + AtomicInteger downloadBytesCount; + + public DownloadThread(Connection conn, int startPos, int endPos) { + this.conn = conn; + this.startPos = startPos; + this.endPos = endPos; + } + + public DownloadThread(Connection conn, int startPos, int endPos, RandomAccessFile ras, AtomicInteger atomicLong) { + this(conn, startPos, endPos); + this.ras = ras; + this.downloadBytesCount = atomicLong; + } + + public void run() { + try { + InputStream is = conn.getInputStream(); + is.skip(startPos); + ras.seek(startPos); + byte[] bytes = new byte[BUFF_SIZE]; + int hasRead = 0; + int readTimes = (endPos - startPos) / BUFF_SIZE + 4; + for (int i = 0; i < readTimes; i++) { + hasRead = is.read(bytes ); + if (hasRead == -1) { + break; + } + ras.write(bytes, 0, hasRead); + downloadBytesCount.getAndAdd(hasRead); + } + + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/group24/494800949/src/main/java/com/coding/week3/download/FileDownloader.java b/group24/494800949/src/main/java/com/coding/week3/download/FileDownloader.java new file mode 100644 index 0000000000..0e40bc38a8 --- /dev/null +++ b/group24/494800949/src/main/java/com/coding/week3/download/FileDownloader.java @@ -0,0 +1,133 @@ +package com.coding.week3.download; + + +import com.coding.week3.download.api.Connection; +import com.coding.week3.download.api.ConnectionException; +import com.coding.week3.download.api.ConnectionManager; +import com.coding.week3.download.api.DownloadListener; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.concurrent.atomic.AtomicInteger; + +public class FileDownloader { + + String url; + + DownloadListener listener; + + ConnectionManager cm; + + String savePath; + + int nThread; + + int fileLength; + + AtomicInteger downloadBytesCount; + + public FileDownloader(String _url) { + this.url = _url; + } + + public FileDownloader(int nThread, String savePath, String url) { + this.nThread = nThread; + this.savePath = savePath; + this.url = url; + downloadBytesCount = new AtomicInteger(0); + } + + public void execute() throws IOException { + // 在这里实现你的代码, 注意: 需要用多线程实现下载 + // 这个类依赖于其他几个接口, 你需要写这几个接口的实现代码 + // (1) ConnectionManager , 可以打开一个连接,通过Connection可以读取其中的一段(用startPos, endPos来指定) + // (2) DownloadListener, 由于是多线程下载, 调用这个类的客户端不知道什么时候结束,所以你需要实现当所有 + // 线程都执行完以后, 调用listener的notifiedFinished方法, 这样客户端就能收到通知。 + // 具体的实现思路: + // 1. 需要调用ConnectionManager的open方法打开连接, 然后通过Connection.getContentLength方法获得文件的长度 + // 2. 至少启动3个线程下载, 注意每个线程需要先调用ConnectionManager的open方法 + // 然后调用read方法, read方法中有读取文件的开始位置和结束位置的参数, 返回值是byte[]数组 + // 3. 把byte数组写入到文件中 + // 4. 所有的线程都下载完成以后, 需要调用listener的notifiedFinished方法 + + // 下面的代码是示例代码, 也就是说只有一个线程, 你需要改造成多线程的。 + Connection conn = null; + try { + + //根据文件长度为每个线程分配下载字节数量 + conn = cm.open(this.url); + fileLength = conn.getContentLength(); + //已读字节数量 + + int perLenOfThread ; + int lastLen ; + Thread[] threads = new Thread[nThread]; + if ( fileLength % nThread == 0) { + perLenOfThread = fileLength / nThread; + } else { + lastLen = fileLength % nThread; + perLenOfThread = (fileLength + (nThread - lastLen)) / nThread; + } + //计算 + + //启动线程 + for (int i = 0; i < nThread; i++) { + Connection conn1 = cm.open(this.url); + RandomAccessFile ras = new RandomAccessFile(savePath, "rw"); + if ( i < nThread - 1) { + threads[i] = new DownloadThread(conn1, perLenOfThread * i, perLenOfThread * (i + 1)-1, ras, downloadBytesCount); + threads[i].start(); + } else { + threads[i] = new DownloadThread(conn1, perLenOfThread * (nThread - 1), fileLength - 1, ras, downloadBytesCount); + threads[i].start(); + } + } + + shutdown(threads); + + } catch (ConnectionException e) { + e.printStackTrace(); + }finally{ + if(conn != null){ + conn.close(); + } + } + } + + private void shutdown(Thread[] threads){ + while (true) { + boolean allTerminated = true; + for (int i = 0; i < nThread; i++) { + allTerminated = allTerminated & threads[i].getState().equals(Thread.State.TERMINATED); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + e.printStackTrace(); + } +// System.out.println(threads[i].getName() + "状态:"+threads[i].getState().toString()); + } + + System.out.println("已下载:"+(downloadBytesCount.get()/1024)+"K,百分比:"+ (downloadBytesCount.get()/ (fileLength/100))+"%" ); + if (allTerminated) { + listener.notifyFinished(); + break; + } + } + } + + + public void setListener(DownloadListener listener) { + this.listener = listener; + } + + + + public void setConnectionManager(ConnectionManager ucm){ + this.cm = ucm; + } + + public DownloadListener getListener(){ + return this.listener; + } + +} diff --git a/group24/494800949/src/main/java/com/coding/week3/download/api/Connection.java b/group24/494800949/src/main/java/com/coding/week3/download/api/Connection.java new file mode 100644 index 0000000000..26261bc920 --- /dev/null +++ b/group24/494800949/src/main/java/com/coding/week3/download/api/Connection.java @@ -0,0 +1,26 @@ +package com.coding.week3.download.api; + +import java.io.IOException; +import java.io.InputStream; + +public interface Connection { + /** + * 给定开始和结束位置, 读取数据, 返回值是字节数组 + * @param startPos 开始位置, 从0开始 + * @param endPos 结束位置 + * @return + */ + byte[] read(int startPos, int endPos) throws IOException; + /** + * 得到数据内容的长度 + * @return + */ + int getContentLength(); + + /** + * 关闭连接 + */ + void close(); + + InputStream getInputStream() throws IOException; +} diff --git a/group24/494800949/src/main/java/com/coding/week3/download/api/ConnectionException.java b/group24/494800949/src/main/java/com/coding/week3/download/api/ConnectionException.java new file mode 100644 index 0000000000..1a4b6a17f2 --- /dev/null +++ b/group24/494800949/src/main/java/com/coding/week3/download/api/ConnectionException.java @@ -0,0 +1,10 @@ +package com.coding.week3.download.api; + +public class ConnectionException extends Exception { + public ConnectionException() { + } + + public ConnectionException(String message) { + super(message); + } +} diff --git a/group24/494800949/src/main/java/com/coding/week3/download/api/ConnectionManager.java b/group24/494800949/src/main/java/com/coding/week3/download/api/ConnectionManager.java new file mode 100644 index 0000000000..9ff27b7432 --- /dev/null +++ b/group24/494800949/src/main/java/com/coding/week3/download/api/ConnectionManager.java @@ -0,0 +1,10 @@ +package com.coding.week3.download.api; + +public interface ConnectionManager { + /** + * 给定一个url , 打开一个连接 + * @param url + * @return + */ + Connection open(String url) throws ConnectionException; +} diff --git a/group24/494800949/src/main/java/com/coding/week3/download/api/DownloadListener.java b/group24/494800949/src/main/java/com/coding/week3/download/api/DownloadListener.java new file mode 100644 index 0000000000..fd43497a10 --- /dev/null +++ b/group24/494800949/src/main/java/com/coding/week3/download/api/DownloadListener.java @@ -0,0 +1,5 @@ +package com.coding.week3.download.api; + +public interface DownloadListener { + void notifyFinished(); +} diff --git a/group24/494800949/src/main/java/com/coding/week3/download/impl/ConnectionImpl.java b/group24/494800949/src/main/java/com/coding/week3/download/impl/ConnectionImpl.java new file mode 100644 index 0000000000..fe9b0a9079 --- /dev/null +++ b/group24/494800949/src/main/java/com/coding/week3/download/impl/ConnectionImpl.java @@ -0,0 +1,55 @@ +package com.coding.week3.download.impl; + +import com.coding.week3.download.api.Connection; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; + + +public class ConnectionImpl implements Connection { + + private HttpURLConnection httpURLConnection; + + public ConnectionImpl() { + } + + public ConnectionImpl(HttpURLConnection httpURLConnection) { + this.httpURLConnection = httpURLConnection; + } + + + @Override + public byte[] read(int startPos, int endPos) throws IOException { + if (startPos < 0 || endPos < 0 || endPos > getContentLength()) { + throw new IndexOutOfBoundsException(); + } + if (startPos >= endPos) { + throw new IllegalArgumentException(); + } + byte[] bytes = new byte[endPos - startPos + 1]; + + InputStream inputStream = httpURLConnection.getInputStream(); + BufferedInputStream bis = new BufferedInputStream(inputStream); + bis.read(bytes, startPos ,bytes.length-1); + return bytes; + } + + + @Override + public int getContentLength() { + return httpURLConnection.getContentLength(); + } + + @Override + public void close() { + httpURLConnection.disconnect(); + } + + @Override + public InputStream getInputStream() throws IOException { + return httpURLConnection.getInputStream(); + } + +} diff --git a/group24/494800949/src/main/java/com/coding/week3/download/impl/ConnectionManagerImpl.java b/group24/494800949/src/main/java/com/coding/week3/download/impl/ConnectionManagerImpl.java new file mode 100644 index 0000000000..3fd451036b --- /dev/null +++ b/group24/494800949/src/main/java/com/coding/week3/download/impl/ConnectionManagerImpl.java @@ -0,0 +1,37 @@ +package com.coding.week3.download.impl; + + +import com.coding.week3.download.api.Connection; +import com.coding.week3.download.api.ConnectionException; +import com.coding.week3.download.api.ConnectionManager; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; + +public class ConnectionManagerImpl implements ConnectionManager { + + @Override + public Connection open(String url) throws ConnectionException { + try { + // 统一资源 + URL realurl = new URL(url); + // 连接类的父类,抽象类 + URLConnection urlConnection = realurl.openConnection(); + // http的连接类 + HttpURLConnection httpURLConnection = (HttpURLConnection) urlConnection; + // 设定请求的方法,默认是GET + httpURLConnection.setRequestMethod("GET"); + httpURLConnection.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)"); + // 设置字符编码 + httpURLConnection.setRequestProperty("Charset", "UTF-8"); + // 打开到此 URL 引用的资源的通信链接(如果尚未建立这样的连接)。 + httpURLConnection.connect(); + return new ConnectionImpl(httpURLConnection); + } catch (java.io.IOException e) { + e.printStackTrace(); + throw new ConnectionException(e.getMessage()); + } + } + +} diff --git a/group24/494800949/src/test/java/com/coding/week3/FileDownloaderTest.java b/group24/494800949/src/test/java/com/coding/week3/FileDownloaderTest.java new file mode 100644 index 0000000000..5b423f145f --- /dev/null +++ b/group24/494800949/src/test/java/com/coding/week3/FileDownloaderTest.java @@ -0,0 +1,103 @@ +package com.coding.week3; + +import com.coding.week3.download.FileDownloader; +import com.coding.week3.download.api.ConnectionManager; +import com.coding.week3.download.api.DownloadListener; +import com.coding.week3.download.impl.ConnectionManagerImpl; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.*; + + +public class FileDownloaderTest { + boolean downloadFinished = false; + static final int DEFAULT_THREADS_NUM = 5; + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testDownload() throws IOException { + +// String url = "http://101.95.48.97:8005/res/upload/interface/apptutorials/manualstypeico/6f83ce8f-0da5-49b3-bac8-fd5fc67d2725.png"; +// String url = "http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-arm32-vfp-hflt.tar.gz"; +// String url = "https://timgsa.baidu.com/timg?image&quality=80&size=b9999_10000&sec=1490808670106&di=48aa6fb7af641f0cb6f9e19120b60c7c&imgtype=0&src=http%3A%2F%2Fwww.ntjoy.com%2Fliv_loadfile%2Fhealth%2Fdzcs%2Fnvr%2Ffold1%2F1360480639_97304600.jpg"; + String url = "https://download.jetbrains.com/idea/ideaIU-2017.1.exe"; + String path = new File("").getAbsolutePath(); + String filename = url.substring(url.lastIndexOf("/"), url.length()); + filename = path +File.separator + filename; + + FileDownloader downloader = new FileDownloader(DEFAULT_THREADS_NUM, filename, url); + ConnectionManager cm = new ConnectionManagerImpl(); + downloader.setConnectionManager(cm); + downloader.setListener(new DownloadListener() { + @Override + public void notifyFinished() { + downloadFinished = true; + } + + }); + + + downloader.execute(); + + // 等待多线程下载程序执行完毕 + while (!downloadFinished) { + try { + System.out.println("还没有下载完成,休眠五秒"); + //休眠5秒 + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + System.out.println("下载完成!"); + + + + } + + + @Test + public void lenTest(){ + int lastLen = 0; + int length = 10232; + int nThread = 10; + int perLenOfThread = 0; + + if ( length % nThread == 0) + perLenOfThread = length / nThread; + else { + lastLen = length % nThread; + perLenOfThread = (length - lastLen) / nThread; + } + Assert.assertEquals(perLenOfThread, 1023); + Assert.assertEquals(lastLen, 2); + + Thread[] threads = new Thread[nThread+1]; + if ( length % nThread == 0) { + perLenOfThread = length / nThread; + } + else { + lastLen = length % nThread; + perLenOfThread = (length - lastLen) / nThread; + } + for (int i = 0; i <= nThread; i++) { + if ( i < nThread) { + System.out.println("startPos: " + perLenOfThread * i); + System.out.println("endPos: " + (perLenOfThread * (i + 1) - 1)); + } else { + System.out.println("startPos: " + perLenOfThread * nThread); + System.out.println("endPos: " + (length - 1)); + } + } + } +}