!pip install BeautifulSoup4
import requests
from bs4 import BeautifulSoup
Collecting BeautifulSoup4
Downloading beautifulsoup4-4.10.0-py3-none-any.whl (97 kB)
|████████████████████████████████| 97 kB 3.2 MB/s
Collecting soupsieve>1.2
Downloading soupsieve-2.3.1-py3-none-any.whl (37 kB)
Installing collected packages: soupsieve, BeautifulSoup4
Successfully installed BeautifulSoup4-4.10.0 soupsieve-2.3.1
WARNING: You are using pip version 20.1.1; however, version 22.0.3 is available.
You should consider upgrading via the '/root/venv/bin/python -m pip install --upgrade pip' command.
URLCONGRESSMEN = 'https://www.congreso.gob.pe/pleno/congresistas/'
#GET ALL NAMES AND URL'S
PLENO_PAGE = requests.get(URLCONGRESSMEN).text
PLENO_SOUP = BeautifulSoup(PLENO_PAGE, 'lxml')
LIST_PLENO = PLENO_SOUP.find_all(class_='conginfo')
LIST_CONGRESSMEN_NAMES = [i.text for i in LIST_PLENO]
LIST_URLCONGINFO = [i['href'] for i in LIST_PLENO]
#GETTING REGIONS
LIST_REGIONES = []
LIST_VOTACION = []
LIST_GRUPO = []
for url in LIST_URLCONGINFO:
CONG_PAGE = requests.get(URLCONGRESSMEN+url).text
CONG_SOUP = BeautifulSoup(CONG_PAGE, 'lxml')
LIST_REGIONES.append(CONG_SOUP.find('p', class_='representa').find(class_='value').text)
LIST_VOTACION.append(CONG_SOUP.find('p', class_='votacion').find(class_='value').text)
LIST_GRUPO.append(CONG_SOUP.find('p', class_='grupo').find(class_='value').text)
import pandas as pd
Congress = pd.DataFrame({"Congresista":LIST_CONGRESSMEN_NAMES,
"Región":LIST_REGIONES,
"Votación":LIST_VOTACION,
"Grupo":LIST_GRUPO})
Congress['Comisiones'] = [[] for i in range(len(Congress['Congresista']))]
urlcomisiones = 'https://www.congreso.gob.pe/CuadrodeComisiones/'
P_comisiones = requests.get(urlcomisiones)
P_comisiones.status_code
soup_comisiones = BeautifulSoup(P_comisiones.text, 'lxml')
tags_comisiones = soup_comisiones.find_all('a', target='_blank')
comisiones = tags_comisiones[:24]
links_comisiones = [i.get('href') for i in comisiones]
nombre_comisiones = [i.text for i in comisiones]
n=0
COMISIONES_MIEMBROS_CARGOS = []
for URL_COMISION in links_comisiones:
nonrawpage = requests.get(URL_COMISION+'/sobrelacomision/integrantes/')
rawsoup = BeautifulSoup(nonrawpage.text, 'lxml')
try:
URL_COMISIONreal = rawsoup.find('iframe', attrs={'name':'ventana02'}).get('src')
except:
print("la interfaz no tiene los integrantes. Comision: "+nombre_comisiones[links_comisiones.index(URL_COMISION)])
PaginaComision = requests.get(URL_COMISIONreal)
SoupComision = BeautifulSoup(PaginaComision.text, 'lxml')
Miembros = [i.get_text() for i in SoupComision.find_all('a', attrs={'class':'conginfo'})]
Cargos = [ i.get_text() for i in SoupComision.find_all('span', attrs={'class':'partidolist'})][1::2]
COMISIONES_MIEMBROS_CARGOS.append({Miembros[i]:[i[0] for i in Cargos][i] for i in range(len(Miembros))})
n+=1
for zip in COMISIONES_MIEMBROS_CARGOS:
for (name, cargo) in zip.items():
listindex=([nombre_comisiones[COMISIONES_MIEMBROS_CARGOS.index(zip)], cargo])
Congress['Comisiones'][Congress.index[Congress['Congresista']==name].tolist()[0]].append(listindex)
#remember that response is the variable that contains URLCONGRESSMEN
soup_partido = BeautifulSoup(PLENO_PAGE, 'lxml')
partidos = [i.get_text() for i in soup_partido.find_all('span', attrs={'class':'partidolist'})]
Congress['Bancada'] = partidos
Congress.to_csv('Congress.csv', index=False)