113 lines
5.3 KiB
Python
113 lines
5.3 KiB
Python
|
from Service import Service
|
||
|
|
from Project import REPO_TYPES, Project
|
||
|
|
from Release import Release
|
||
|
|
from Issue import IssueComment, Issue
|
||
|
|
from Wiki import Wiki
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
|
||
|
|
|
||
|
|
class googlecode(Service):
|
||
|
|
|
||
|
|
DOMAIN = "https://code.google.com"
|
||
|
|
|
||
|
|
# Since I want to stay on Google's good side
|
||
|
|
# I'm going to write this method to parse a single project
|
||
|
|
# You will need to provide your own project list to roll through
|
||
|
|
# Such a list exists (although incomplete)
|
||
|
|
# http://flossdata.syr.edu/data/gc/2012/2012-Nov/gcProjectInfo2012-Nov.txt.bz2
|
||
|
|
def getProject(self, projectName):
|
||
|
|
project = Project()
|
||
|
|
sourceType = None
|
||
|
|
projectURL = self.DOMAIN + "/p/" + projectName + "/"
|
||
|
|
|
||
|
|
projectpageHTML = self.curl_get(projectURL).getvalue()
|
||
|
|
projectpageSoup = BeautifulSoup(projectpageHTML, "html.parser")
|
||
|
|
|
||
|
|
sourceURL = projectpageSoup.find(name="a", string="Source").get("href")
|
||
|
|
sourceSoup = BeautifulSoup(self.curl_get(self.DOMAIN + "/p/" + sourceURL).getvalue(), "html.parser")
|
||
|
|
sourceSoupText = sourceSoup.get_text()
|
||
|
|
|
||
|
|
# get source
|
||
|
|
if "git clone" in sourceSoupText:
|
||
|
|
project.repoType = REPO_TYPES.git
|
||
|
|
project.repoURL = "https://code.google.com/p/" + projectName + "/"
|
||
|
|
elif "svn co" in sourceSoupText:
|
||
|
|
project.repoType = REPO_TYPES.SVN
|
||
|
|
project.repoURL = "http://" + projectName + ".googlecode.com/svn/"
|
||
|
|
else:
|
||
|
|
project.repoType = REPO_TYPES.hg
|
||
|
|
project.repoURL = "https://code.google.com/p/" + projectName + "/"
|
||
|
|
|
||
|
|
|
||
|
|
# get downloads
|
||
|
|
project.releases = []
|
||
|
|
downlaodsSoup = BeautifulSoup(self.curl_get(projectURL + "downloads/list").getvalue(), "html.parser")
|
||
|
|
downloadSection = downlaodsSoup.find("table", "results")
|
||
|
|
if "Your search did not generate any results." not in downlaodsSoup.get_text():
|
||
|
|
downloadRows = downloadSection.find_all("tr")[1:]
|
||
|
|
for downloadRow in downloadRows:
|
||
|
|
cols = downloadRow.find_all("td")
|
||
|
|
downloadTD = cols[1]
|
||
|
|
downloadURL = "https://" + projectName + ".googlecode.com/files/" + downloadTD.a.text.replace("\n", "").strip(" ")
|
||
|
|
fileName = downloadTD.a.text.replace("\n", "").strip(" ")
|
||
|
|
release = Release()
|
||
|
|
release.fileURL = downloadURL
|
||
|
|
release.fileName = fileName
|
||
|
|
project.releases.append(release)
|
||
|
|
|
||
|
|
# get issues
|
||
|
|
project.issues = []
|
||
|
|
issuesSoup = BeautifulSoup(self.curl_get(projectURL + "issues/list").getvalue(), "html.parser")
|
||
|
|
if "Your search did not generate any results." not in issuesSoup.get_text():
|
||
|
|
issuesSection = issuesSoup.find("table", "results")
|
||
|
|
for issueRow in issuesSection.find_all("tr")[1:]:
|
||
|
|
issue = Issue()
|
||
|
|
cols = issueRow.find_all("td")
|
||
|
|
issueId = cols[1].text.replace("\n", "").strip()
|
||
|
|
issueURL = projectURL + "issues/detail?id=" + issueId
|
||
|
|
issueStatus = cols[3].text.replace("\n", "").strip(" ")
|
||
|
|
issueSummary = cols[8].text.replace("\n", "")
|
||
|
|
issueTitle = cols[8].text.replace("\n", "")
|
||
|
|
issueAuthor = cols[5].text.replace("\n", "")
|
||
|
|
|
||
|
|
#issue.author = issueAuthor
|
||
|
|
issue.comments = []
|
||
|
|
issue.status = issueStatus.strip(" ")
|
||
|
|
issue.summary = issueSummary.strip(" ")
|
||
|
|
issue.title = issueTitle
|
||
|
|
issue.id = issueId
|
||
|
|
|
||
|
|
# we must go deeper to get comments
|
||
|
|
issueComments = BeautifulSoup(self.curl_get(issueURL).getvalue(), "html.parser")
|
||
|
|
for comment in issueComments.find_all("div", "vt"):
|
||
|
|
#author = comment.find(class_="author").find("a").text
|
||
|
|
author = (comment.find(class_="author").find_all("a")[-1]).contents
|
||
|
|
date = comment.find("span", "date")["title"]
|
||
|
|
commentText = comment.find("pre").get_text()
|
||
|
|
issueComment = IssueComment()
|
||
|
|
issueComment.date = date
|
||
|
|
issueComment.author = author
|
||
|
|
issueComment.summary = commentText
|
||
|
|
issue.comments.append(issueComment)
|
||
|
|
|
||
|
|
project.issues.append(issue)
|
||
|
|
|
||
|
|
# get wiki pages
|
||
|
|
project.wikis = []
|
||
|
|
wikiSoup = BeautifulSoup(self.curl_get(projectURL + "w/list").getvalue(), "html.parser")
|
||
|
|
if "Your search did not generate any results." not in wikiSoup.get_text():
|
||
|
|
wikiSection = wikiSoup.find("table", "results")
|
||
|
|
for wikiRow in wikiSection.find_all("tr")[1:]:
|
||
|
|
wiki = Wiki()
|
||
|
|
cols = wikiRow.find_all("td")
|
||
|
|
wiki.pageName = cols[1].text.replace("\n", "").strip(" ")
|
||
|
|
wiki.summary = cols[2].text.replace("\n", "").strip(" ")
|
||
|
|
wiki.updated = cols[3].text.replace("\n", "").strip(" ")
|
||
|
|
wikiURL = projectURL + "wiki/" + wiki.pageName
|
||
|
|
wikiPageSoup = BeautifulSoup(self.curl_get(wikiURL).getvalue(), "html.parser")
|
||
|
|
wikiContent = wikiPageSoup.find(id="wikicontent")
|
||
|
|
wiki.htmlContent = wikiContent.prettify()
|
||
|
|
wiki.textContent = wikiContent.get_text()
|
||
|
|
project.wikis.append(wiki)
|
||
|
|
|
||
|
|
return project
|