上一篇文章的地址是：http://www.jb51.cc/article/p-eadxdfhy-bad.html
这次带来的是一个 当当网 的爬虫app。
先放出GitHub链接：
https://github.com/CallMeSp/DangDang
先看一下结构分析图：
功能不复杂，但是有Activity、Adapter、自定义view、MVP结构、sqlite等一个app有的几乎一切。
功能实现过程说明：
下面进入正文：
首先这是一个爬虫，要先建立我们的model类
class Bookitem (var map:MutableMap<String,Any?>){
var name: String by map
var detail: String by map
var price: String by map
var author: String by map
var imgurl: String by map
var publisher: String by map
var detailurl:String by map
constructor(name :String,detail:String,price:String,author:String,imgurl:String,publisher:String,detailurl:String):this(HashMap()){
this.name=name
this.detail=detail
this.price=price
this.author=author
this.imgurl=imgurl
this.publisher=publisher
this.detailurl=detailurl
}
}
本来单纯建立这样一个类是并不需要这个Map的，但是这里面是为了后面的sqlite操作做铺垫，anko框架中数据库存储和读取值得时候的映射就是要通过这个Map来实现的，要注意的是这里面每个变量的名字要和数据库Table中对应的列的名字一样。
createTable(tbname,true,"id" to INTEGER+ PRIMARY_KEY+ AUTOINCREMENT,"name" to TEXT,"price" to TEXT,"author" to TEXT,"publisher" to TEXT,"detail" to TEXT,"imgurl" to TEXT,"detailurl" to TEXT)
下面就来看一下关键的几个功能实现：
1.根据书名和页码获取列表
fun getBookListByNameAndPage(bookname: String,page: Int,ismore: Boolean) {
Observable.just(bookname)
.observeOn(Schedulers.newThread())
.map { s ->
mainpresenter.showPB()
val bookitemArrayList = ArrayList<Bookitem>()
val doc = Jsoup.connect("http://search.dangdang.com/?key=$s&act=input&page_index=$page").get()
Log.e("..",doc.toString())
val elements = doc.select("ul.bigimg").select("li")
for (element in elements) {
val name = element.select("a").attr("title")
val detailurl = element.select("a").attr("href")
val imgurl = if (!element.select("a").select("img").attr("src").startsWith("http"))
element.select("a").select("img").attr("data-original")
else
element.select("a").select("img").attr("src")
val detail = element.select("p.detail").text()
val price = element.select("p.price").select("span.search_now_price").text()
val author = element.select("p.search_book_author").select("span")[0].select("a").attr("title")
val publisher = element.select("p.search_book_author").select("span")[2].select("a").text()
val bookitem = Bookitem(name,detail,price,author,imgurl,publisher,detailurl)
Log.e(TAG,"detailurl:"+detailurl)
bookitemArrayList.add(bookitem)
}
bookitemArrayList
}
.subscribe(object : DisposableObserver<ArrayList<Bookitem>>() {
override fun onNext(bookitems: ArrayList<Bookitem>) {
if (!ismore) {
mainpresenter.updateUI(bookitems)
} else {
mainpresenter.addMoreList(bookitems)
}
}
override fun onError(e: Throwable) {
mainpresenter.updateUI(ArrayList<Bookitem>())
mainpresenter.hidePB()
}
override fun onComplete() {
mainpresenter.hidePB()
}
})
}
2.书籍详情界面获取procontent
fun getDetail(url:String){
Observable.just(url)
.observeOn(Schedulers.newThread())
.subscribe(object :DisposableObserver<String>(){
override fun onError(e: Throwable?) {
}
override fun onComplete() {
}
override fun onNext(t: String?) {
val detailurl = url
Log.e(TAG,"url?:" + t)
val doc = Jsoup.connect(detailurl).get()
Log.e("??",doc.toString())
var elements_content = doc.select("div.t_Box").select("div.t_Box_left")
var procontent = doc.select("div.t_Box").select("div.t_Box_left").select("div.pro_content").select("ul").select("li")
var contents = doc.select("div.section")
var catalog = doc.select("div.section")
var stb:StringBuffer= StringBuffer()
//procontent加入换行符
for (str:Element in procontent){
stb.append(str.text())
stb.append("\n")
}
detailpresenter.setProContent(stb.toString())
Log.e("procontent",stb.toString())
}
})
}
3.DbHelper
class DbHelper(context:Context):ManagedsqliteOpenHelper(context,DB_NAME,null,DB_VERSION){
val mcontext=context
@Volatile var TB_NAME:String=""
companion object {
val DB_NAME="history.db"
val DB_VERSION=1
val TAG="DbHelper"
//val instance by lazy{ DbHelper()}
}
override fun onCreate(db: sqliteDatabase?) {
Log.e(TAG,"onCreate")
}
fun insertNewTable(tbname:String,booklist:ArrayList<Bookitem>){
DbHelper(mcontext).use {
/*createTable(tbname,true,"id" to INTEGER+ PRIMARY_KEY+ AUTOINCREMENT,"bookname" to TEXT,"price" to TEXT,"author" to TEXT,"publisher" to TEXT,"detail" to TEXT,"imgurl" to TEXT,"detailurl" to TEXT)*/
execsql("create table if not exists "+tbname+"(it INTEGER primary key AUTOINCREMENT," +
"name text," +
"detail text," +
"price text," +
"author text,"
+"imgurl text," +
"publisher text," +
"detailurl text)")
Log.e(TAG,"create table:"+tbname)
booklist.forEach { insert(tbname,"name" to it.name,"price" to it.price,"author" to it.author,"publisher" to it.author,"detail" to it.detail,"imgurl" to it.imgurl,"detailurl" to it.detailurl)
Log.e(TAG,"insert into table:"+tbname+": "+it.name)}
}
}
fun addToOldTable(tbname:String,booklist:ArrayList<Bookitem>) {
DbHelper(mcontext).use {
booklist.forEach {
insert(tbname,"insert into table:"+tbname+": "+it.name)
}
}
}
fun getListFormDb(bookname:String)= DbHelper(mcontext).use {
select(bookname).parseList {
Bookitem(HashMap(it))
}
}
fun dropTable(bookname: String)=DbHelper(mcontext).use {
clear(bookname)
}
fun IshaveTB(bookname:String):Boolean=DbHelper(mcontext).use {
var result:Boolean=false
var cursor=rawQuery("select count(*) as c from sqlite_master where type ='table' and name ="+"\'"+bookname+"\';",null)
if (cursor.moveToNext()){
var count:Int=cursor.getInt(0)
if (count>0){
result=true
}
}
result
}
override fun onUpgrade(db: sqliteDatabase?,oldVersion: Int,newVersion: Int) {
db!!.dropTable(TB_NAME,true)
onCreate(db)
}
fun <T : Any> SelectQueryBuilder.parseList(parser: (Map<String,Any?>) -> T): List<T> =
parseList(object : MapRowParser<T> {
override fun parseRow(columns: Map<String,Any?>): T = parser(columns)
})
fun <T : Any> SelectQueryBuilder.parSEOpt(parser: (Map<String,Any?>) -> T): T? =
parSEOpt(object : MapRowParser<T> {
override fun parseRow(columns: Map<String,Any?>): T = parser(columns)
})
fun sqliteDatabase.clear(tableName: String) {
execsql("delete from $tableName")
}
}
4.上拉下拉监听：
class SwipeRefreshView(context: Context,attrs: AttributeSet) : SwipeRefreshLayout(context,attrs) {
private val mScaledTouchSlop: Int
private val mFooterView: View
private var recyclerview: RecyclerView? = null
@Volatile private var condition2: Boolean = false
private var MonLoadListener:()->Unit={}
/** * 正在加载状态 */
private var isLoading: Boolean = false
internal var layoutInflater: LayoutInflater
init {
// 填充底部加载布局
mFooterView = View.inflate(context,R.layout.view_footer,null)
// 表示控件移动的最小距离，手移动的距离大于这个距离才能拖动控件
mScaledTouchSlop = ViewConfiguration.get(context).scaledTouchSlop
layoutInflater = LayoutInflater.from(context)
}
override fun onLayout(changed: Boolean,left: Int,top: Int,right: Int,bottom: Int) {
super.onLayout(changed,left,top,right,bottom)
// recyclerview,设置recyclerview的布局位置
if (recyclerview == null) {
// 判断容器有多少个孩子
if (childCount > 0) {
// 判断第一个孩子是不是ListView
if (getChildAt(0) is RecyclerView) {
// 创建ListView对象
recyclerview = getChildAt(0) as RecyclerView
// 设置ListView的滑动监听
setRecyclerViewOnScroll()
}
}
}
}
/** * 在分发事件的时候处理子控件的触摸事件 * @param ev * * * @return */
private var mDownY: Float = 0.toFloat()
private var mUpY: Float = 0.toFloat()
override fun dispatchTouchEvent(ev: MotionEvent): Boolean {
when (ev.action) {
MotionEvent.ACTION_DOWN ->
// 移动的起点
mDownY = ev.y
MotionEvent.ACTION_MOVE ->
// 移动过程中判断时候能下拉加载更多
if (condition2 && canLoadMore()) {
// 加载数据
loadData()
return false
}
MotionEvent.ACTION_UP ->
// 移动的终点
mUpY = y
}
return super.dispatchTouchEvent(ev)
}
/** * 判断是否满足加载更多条件 * @return */
private fun canLoadMore(): Boolean {
// 1. 是上拉状态
val condition1 = mDownY - mUpY >= mScaledTouchSlop
if (condition1) {
println("是上拉状态")
}
// 3. 正在加载状态
val condition3 = !isLoading
if (condition3) {
println("不是正在加载状态")
}
return condition1 && condition3
}
/** * 处理加载数据的逻辑 */
private fun loadData() {
println("加载数据...")
if (MonLoadListener != null) {
// 设置加载状态，让布局显示出来
setLoading(true)
MonLoadListener()
}
}
/** * 设置加载状态，是否加载传入boolean值进行判断 * @param loading */
fun setLoading(loading: Boolean) {
// 修改当前的状态
isLoading = loading
}
/** * 设置RecyclerView的滑动监听 */
private fun setRecyclerViewOnScroll() {
recyclerview!!.setOnScrollChangeListener { v,scrollX,scrollY,oldScrollX,oldScrollY ->
val layoutManager = recyclerview!!.layoutManager
val linearManager = layoutManager as LinearLayoutManager
val lastItemPosition = linearManager.findLastVisibleItemPosition()
if (recyclerview!!.adapter.itemCount - 1 == lastItemPosition) {
condition2 = true
} else {
condition2 = false
}
}
}
fun setCondition2(isLoading: Boolean) {
condition2 = isLoading
}
/** * 上拉加载的接口回调 */
fun setMOnLoadListener(listener:()->Unit){
MonLoadListener=listener
}
companion object {
private val TAG = "SwipeRefreshView"
}
}
要注意的是这里的DBhelper中的扩展函数。
如：
fun dropTable(bookname: String)=DbHelper(mcontext).use {
clear(bookname)
}
