Q:

Parsing json element

I am using Scrapy and a Regex to parse some none standard web source code. I then wish to parse the first element of the dictionary returned:
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import Selector
from scrapy.item import Item
from scrapy.spider import BaseSpider
from scrapy import log
from scrapy.cmdline import execute
from scrapy.utils.markup import remove_tags
import time
import re
import json
import requests

class ExampleSpider(CrawlSpider):
    name = "goal2"
    allowed_domains = ["whoscored.com"]
    start_urls = ["http://www.whoscored.com"]
    download_delay = 5

    rules = [Rule(SgmlLinkExtractor(allow=('\Teams'),deny=(),), follow=False, callback='parse_item')]

    def parse_item(self, response):

        sel = Selector(response)
        titles = sel.xpath("normalize-space(//title)")
        print '-' * 170
        myheader = titles.extract()[0]
        print '********** Page Title:', myheader.encode('utf-8'), '**********'
        print '-' * 170

        match1 = re.search(re.escape("DataStore.prime('stage-player-stat', defaultTeamPlayerStatsConfigParams.defaultParams , ") \
                     + '(\[.*\])' + re.escape(");"), response.body)

        if match1 is not None:
            playerdata1 = match1.group(1)

            teamid = json.loads(playerdata1[0])
            print teamid

The key for the first element of 'playerdata1' is called 'TeamId'. I assumed the above method would work, however I am getting the following error:
    teamid = json.loads(playerdata1[0])
  File "C:\Python27\lib\json\__init__.py", line 338, in loads
    return _default_decoder.decode(s)
  File "C:\Python27\lib\json\decoder.py", line 366, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "C:\Python27\lib\json\decoder.py", line 382, in raw_decode
    obj, end = self.scan_once(s, idx)
exceptions.ValueError: Expecting object: line 1 column 1 (char 0)

Can anyone see what the issue is here?

A:

match1.group(1) returns one string. You then index that string:
teamid = json.loads(playerdata1[0])

Here, [0] will give you the just the first character of that string. Remove the indexing expression there to use the whole string:
teamid = json.loads(playerdata1)

Now teamid is a list with player objects:
>>> len(teamid)
22
>>> teamid[0].keys()
[u'FirstName', u'LastName', u'KnownName', u'Field', u'GameStarted', u'AerialWon', u'TeamRegionCode', u'SecondYellow', u'ShotsBlocked', u'TotalShots', u'Assists', u'Red', u'Name', u'PositionText', u'Ranking', u'PositionLong', u'PlayerId', u'SubOff', u'Dispossesed', u'TeamId', u'TotalTackles', u'TotalLongBalls', u'Goals', u'SubOn', u'WasDribbled', u'AerialLost', u'Turnovers', u'ShotsOnTarget', u'WSName', u'Fouls', u'ManOfTheMatch', u'Height', u'TeamName', u'RegionCode', u'TotalPasses', u'TotalThroughBalls', u'Dribbles', u'DateOfBirth', u'OwnGoals', u'WasFouled', u'TotalClearances', u'Rating', u'PlayedPositionsRaw', u'Weight', u'AccurateLongBalls', u'OffsidesWon', u'AccuratePasses', u'Yellow', u'KeyPasses', u'TotalCrosses', u'AccurateCrosses', u'IsCurrentPlayer', u'Age', u'PositionShort', u'AccurateThroughBalls', u'Interceptions', u'Offsides']

