jonjoongtalk

from google.colab import drive drive.mount('/content/drive')

import sys !apt-get update !apt install chromium-chromedriver !cp /usr/lib/chromium-browser/chromedriver /usr/bin !pip install selenium # set options to be headless, .. from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver import ActionChains from bs4 import BeautifulSoup as bs import re import time

import pprint

options = webdriver.ChromeOptions() options.add_argument('--headless') options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage')

top_url = 'https://play.google.com/store/apps/collection/topgrossing?clp=ChMKEQoLdG9wZ3Jvc3NpbmcQBxgD:S:ANO1ljJpBI4&gsr=ChUKEwoRCgt0b3Bncm9zc2luZxAHGAM%3D:S:ANO1ljJ_fZk&hl=ko' wd = webdriver.Chrome('chromedriver',options=options) wd.get(top_url) wd.implicitly_wait(2) for _ in range(2) : wd.execute_script("window.scrollTo(0, document.body.scrollHeight);") time.sleep(1) html = bs(wd.page_source, 'html.parser') # display (html) wd.close() apps = html.find_all('div', {'class' : 'b8cIId ReQCgd Q9MA7b'}) # apps apps_list = [] for a in apps : apps_list.append([a.text, a.find('a')['href'], ]) apps_list # url

len(apps_list)

import pandas as pd df_AppList = pd.DataFrame(apps_list) df_AppList

'http://en.mobileindex.com/app/?code=com.kakao.talk&mk=12&c=kr'.index('=')

wd = webdriver.Chrome('chromedriver', options=options) url = 'http://en.mobileindex.com/app/rank' wd.get(url) wd.implicitly_wait(2) apps_list = [] for idx, app in enumerate(wd.find_elements_by_class_name('appTitle')) : if (idx+1) % 3 == 0 : title = app.find_element_by_class_name('appname').text link = (app.find_element_by_tag_name('a').get_attribute("href")[36:])[:-11] apps_list.append([title, link]) display (apps_list) wd.close()

len(apps_list)

apps_list[0][1]

url_prefix = 'https://play.google.com/store/apps/details?hl=ko&showAllReviews=true&id=' url_full = url_prefix + apps_list[0][1] url_full

'별표 5개 만점에 4개를 받았습니다.'.index('4')

wd = webdriver.Chrome('chromedriver', options=options) wd.get(url_full) wd.implicitly_wait(2) review_list = [] more_nums = 0 while True : # more_nums += 1 wd.execute_script('window.scrollTo(0, document.body.scrollHeight);') # wd.implicitly_wait(1) time.sleep(1) more_nums +=1 print (more_nums, end='\t') if more_nums >30 : break try : # more_btn = wd.find_element_by_xpath('//*[@id="fcxH9b"]/div[4]/c-wiz/div/div[2]/div/div[1]/div/div/div[1]/div[2]/div[2]/div') # if more_nums % 4 == 0 : # more_btn = wd.find_element_by_class_name('M9Bg4d') # ('#fcxH9b > div.WpDbMd > c-wiz > div > div.ZfcPIb > div > div.JNury.Ekdcne > div > div > div.W4P4ne > div:nth-child(2) > div.PFAhAf > div') # more_btn = wd.find_element_by_class_name('U26fgb O0WRkf oG5Srb C0oVfc n9lfJ M9Bg4d') more_btn = wd.find_element_by_xpath('/html/body/div[1]/div[4]/c-wiz/div/div[2]/div/div[1]/div/div/div[1]/div[2]/div[2]/div') print (more_btn.text, end='\t') if more_nums % 2 == 1 : # more_btn = wd.find_element_by_xpath('//*[@id="fcxH9b"]/div[4]/c-wiz/div/div[2]/div/div[1]/div/div/div[1]/div[2]/div[2]/div/span/span') # more_btn = wd.find_element_by_class_name('PFAhAf').find_element_by_class_name('RveJvd snByac') # print (more_btn.text ) ActionChains(wd).click(more_btn).perform() # more_btn.click # print (more_btn.size) print ('pass to..') # break except Exception: print ('re..') # continue for idx, one in enumerate(wd.find_elements_by_class_name('zc7KVe')) : # print (idx) if idx % 2 ==0 : # writer = one.find_element_by_class_name('X43Kjb').text # print (writer) try : star = one.find_element_by_class_name('pf5lIe').find_element_by_tag_name('div').get_attribute('aria-label')[10] # print (star) review = one.find_element_by_class_name('UD7Dzf').text # display (review) review_list.append([int(idx/2), star, review]) except : print (f"error __{int(idx/2)}") review_list.append([int(idx/2), "error", "error",]) # display (review_list) wd.close()

more_btn

wd.find_element_by_class_name('PFAhAf').find_element_by_class_name('CwaK9').text

len(wd.find_elements_by_class_name('zc7KVe'))

wd.find_elements_by_class_name('zc7KVe')[2].find_element_by_class_name('X43Kjb').text

review_list = [] for idx, one in enumerate(wd.find_elements_by_class_name('zc7KVe')) : print (idx) writer = one.find_element_by_class_name('X43Kjb').text print (writer) star = one.find_element_by_class_name('pf5lIe').find_element_by_tag_name('div').get_attribute('aria-label') print (star) # review = one.find_element_by_class_name('UD7Dzf').text # display (review) review_list.append([idx, star, review])

import pandas as pd df_review_list = pd.DataFrame(review_list, columns=['idx', 'star', 'review']) df_review_list.set_index('idx', inplace=True) display(df_review_list)

df_review_list.to_csv('kakao_review.csv')

def get_top50() : print ("get top 50 app list from mobileindex.com") wd = webdriver.Chrome('chromedriver', options=options) url = 'http://en.mobileindex.com/app/rank' wd.get(url) wd.implicitly_wait(2) apps_list = [] for idx, app in enumerate(wd.find_elements_by_class_name('appTitle')) : if (idx+1) % 3 == 0 : title = app.find_element_by_class_name('appname').text link = (app.find_element_by_tag_name('a').get_attribute("href")[36:])[:-11] apps_list.append([title, link]) # display (apps_list) wd.close() return apps_list def review_crawler (app_name, app_url) : print ("app_name : ", app_name, end='\t') print ("app_url : ", app_url) with webdriver.Chrome('chromedriver', options=options) as wd : full_url = 'https://play.google.com/store/apps/details?hl=ko&showAllReviews=true&id=' + app_url wd.get(full_url) wd.implicitly_wait(2) review_list = [] print (f"let's looping!") page_length_list = [] for _ in range(40) : print (f"{_+1}", end=' ') wd.execute_script('window.scrollTo(0, document.body.scrollHeight);') time.sleep(0.5) elements = wd.find_elements_by_class_name('zc7KVe') page_length = len(elements) page_length_list.append(page_length ) print (len(page_length_list), page_length) if len(page_length_list) > 4 and page_length_list[-1] == page_length_list[-4] : break try : more_btn = wd.find_element_by_xpath('/html/body/div[1]/div[4]/c-wiz/div/div[2]/div/div[1]/div/div/div[1]/div[2]/div[2]/div') if _ % 2 == 1 : ActionChains(wd).click(more_btn).perform() except Exception as e : print(e) continue print ("\nget items... ") print (f"writing ...", end=' ') try : for idx, one in enumerate(elements) : if idx % 200 == 0 : print (f"no. {idx//2:4d}", end=' ') if idx % 2 == 0 : star = one.find_element_by_class_name('pf5lIe').find_element_by_tag_name('div').get_attribute('aria-label')[10] review = one.find_element_by_class_name('UD7Dzf').text review_list.append([int(idx/2), star, review]) except Exception as e : print (f"error __{int(idx/2)} {e }") review_list.append([int(idx/2), "error", "error",]) print ('\nmake csv file ...') pd.DataFrame(review_list, columns=['idx', 'star', 'review']).set_index('idx').to_csv(f"/content/drive/My Drive/Programming/1. Docker/VSCode_/Java_Spring/Python/MLDL/ToyProject/{app_name[:5]}.csv") print (f"\n{app_name} reviews crawling finished!\n") return None # return [app_name, review_list] # print(review_crawler(apps_list[30][0], apps_list[30][1])) apps_list = [ ['V LIVE', 'com.naver.vapp'] ] # apps_list = get_top50() reviews_list = [['app_name', 'app_reviews']] # for app in apps_list[17:] : review_crawler(app[0], app[1])

apps_list.index( ['여보야 - 결혼, 재혼을 위한 중매쟁이 앱', 'com.yeoboya.com.yeoboya'] )

import pandas as pd df_reviews_list = pd.DataFrame(reviews_list, columns=['app_name', 'app_reviews']) df_reviews_list.set_index('app_name', inplace=True) display(df_reviews_list)

df_reviews_list.iloc[6]

reviews_list

import sys !apt-get update !apt install chromium-chromedriver !cp /usr/lib/chromium-browser/chromedriver /usr/bin !pip install selenium # set options to be headless, .. from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver import ActionChains options = webdriver.ChromeOptions() options.add_argument('--headless') options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') # from bs4 import BeautifulSoup as bs import re import time import pandas as pd def review_crawler (app_name, app_url) : print ("app_name : ", app_name, end='\t') print ("app_url : ", app_url) with webdriver.Chrome('chromedriver', options=options) as wd : full_url = 'https://play.google.com/store/apps/details?hl=ko&showAllReviews=true&id=' + app_url wd.get(full_url) wd.implicitly_wait(2) review_list = [] print (f"let's looping!") page_length_list = [] for _ in range(40) : print (f"{_+1}", end=' ') wd.execute_script('window.scrollTo(0, document.body.scrollHeight);') time.sleep(0.5) elements = wd.find_elements_by_class_name('zc7KVe') page_length = len(elements) page_length_list.append(page_length ) print (len(page_length_list), page_length) if len(page_length_list) > 4 and page_length_list[-1] == page_length_list[-4] : break try : more_btn = wd.find_element_by_xpath('/html/body/div[1]/div[4]/c-wiz/div/div[2]/div/div[1]/div/div/div[1]/div[2]/div[2]/div') if _ % 2 == 1 : ActionChains(wd).click(more_btn).perform() except Exception as e : print(e) continue print ("\nget items... ") print (f"writing ...", end=' ') try : for idx, one in enumerate(elements) : if idx % 200 == 0 : print (f"no. {idx//2:4d}", end=' ') if idx % 2 == 0 : star = one.find_element_by_class_name('pf5lIe').find_element_by_tag_name('div').get_attribute('aria-label')[10] review = one.find_element_by_class_name('UD7Dzf').text review_list.append([int(idx/2), star, review]) except Exception as e : print (f"error __{int(idx/2)} {e }") review_list.append([int(idx/2), "error", "error",]) print ('\nmake csv file ...') pd.DataFrame(review_list, columns=['idx', 'star', 'review']).set_index('idx').to_csv(f"/content/drive/My Drive/Programming/1. Docker/VSCode_/Java_Spring/Python/MLDL/ToyProject/{app_name[:5]}.csv") print (f"\n{app_name} reviews crawling finished!\n") return None # return [app_name, review_list] # print(review_crawler(apps_list[30][0], apps_list[30][1])) apps_list = [ ['V LIVE', 'com.naver.vapp'] ] # apps_list = get_top50() # reviews_list = [['app_name', 'app_reviews']] for app in apps_list : review_crawler(app[0], app[1])