In this article
Import various libraries being used by this script
import requests as req
import json
import numpy
import pandas as pd
import quantipy as qp
Configuration of REST api, client ID/secret, survey ID, SmartHub ID etc (replace with actual values)
# Configuration
idp_url = "https://idp.[euro|us|aus|nordic].confirmit.com/"
site_url = "https://ws.[euro|us|aus|nordic].confirmit.com/"
client_id = "aa55d014-829c-454b-aaf9-dc4549d9fa80"
client_secret = "788e2edd-6084-4094-9b9d-1e0dc1979129"
# Source configuration
surveyid = "p12345" #Webinar Demo: Relationship survey
# Target configuration
hubid = "1234" #https://author.[euro|us|aus|nordic].confirmit.com/hub/1234/overview
hub_tableid_basic = "2" # basic_weights
hub_tableid_advanced = "3" # advanced_weights
hub_tableid_cellweights = "4" # cell_weights
hub_tableid_interlocked = "5" # interlocked_weights
Alternative 1: Retrieve data using the API
First you must call the identity service endpoint to get a valid token using the Client ID/Secret.
See Security for more information.
Authenticate
# Get access token
response = req.post(idp_url + 'identity/connect/token',
data = "grant_type=api-user&scope=pub.surveys pub.hubs",
auth=(client_id, client_secret),
headers = {'Content-Type': 'application/x-www-form-urlencoded'})
response.raise_for_status()
resp_obj = response.json()
token = resp_obj['access_token']
Read Data
# Get source data records
headers = {'Authorization': 'Bearer ' + source_token, "Accept": "application/json", "Content-Type": "application/json"}
url = site_url + 'v1/surveys/' + surveyid + '/responses/data'
response = req.get(url, data = None, headers = headers, stream=False)
response.raise_for_status()
# Decode json response
res = response.content.decode("utf-8")
json_lines = res.splitlines()
json_data = []
for line in json_lines:
json_data.append(json.loads(line))
# Normalize the data in a pandas dataframe
df = pd.io.json.json_normalize(json_data)
Transform data
# Read data from pandas dataframe into quantipy dataset
dataset = qp.DataSet(name='test', dimensions_comp=True)
# Column 'name' is not allowed by quantipy
dataset.from_components(df.drop(columns=['name']))
# All columns are of type string. convert appropriate columns to singles to allow weighting
dataset.convert('fiscal_year', 'single')
dataset.convert('gender', 'single')
dataset.convert('age_group', 'single')
dataset.convert('region', 'single')
dataset.convert('continent', 'single')
Alternative 2: Retrieve data using Quantipy
dataset = qp.DataSet("test")
dataset.read_confirmit_api(surveyid, site_url, idp_url, client_id, client_secret)
Weighting example - BASIC
# weighting basic rim weights
basicScheme = qp.Rim('test_basic')
# Targets
gender_targets = {}
gender_targets['gender'] = {1: 30, 2: 70}
age_group_targets = {}
age_group_targets['age_group'] = {1: 10, 2: 15, 3:10, 4: 35, 5: 30}
region_targets = {}
region_targets = {'region': {1: 10, 2: 15, 3:15, 4: 25, 5: 15, 6: 10,7: 10}}
all_targets = [gender_targets, age_group_targets, region_targets]
basicScheme.set_targets(targets=all_targets, group_name='basic weights')
basicScheme.groups['basic weights']['targets']
# Calculate weights
responseweights_basic = dataset.weight(basicScheme, weight_name='weight', unique_key='responseid', inplace=False)
responseweights_basic = responseweights_basic.reindex(['responseid', 'weight'],axis='columns')
Weighting example - ADVANCED
# Weighting with different weights within groups, and weighting on groups themselves
advancedScheme = qp.Rim('test_advanced')
# Targets within groups
gender_target_fy1 = {'gender': {1:40, 2:60}}
gender_target_fy2and3 = {'gender': {1:55, 2:45}}
AgeGroup_target_fy1 = {'age_group': {1: 10, 2: 15, 3:10, 4: 35, 5: 30}}
AgeGroup_target_fy2and3 = {'age_group': {1: 20, 2: 10, 3:15, 4: 20, 5: 35}}
region_target_fy1 = {'region': {1: 5, 2: 10, 3:5, 4: 25, 5: 30, 6: 10, 7: 15}}
region_target_fy2and3 = {'region': {1: 10, 2: 15, 3:15, 4: 25, 5: 15, 6: 10,7: 10}}
all_targets_fy1 = [gender_target_fy1, AgeGroup_target_fy1, region_target_fy1]
all_targets_fy2and3 = [gender_target_fy2and3, AgeGroup_target_fy2and3, region_target_fy2and3]
# Add fiscal year groups
filter_fy1 = 'fiscal_year == 1'
filter_fy2and3 = 'fiscal_year == 2 or fiscal_year == 3'
advancedScheme.add_group(name='fiscal year 1', filter_def=filter_fy1, targets=all_targets_fy1)
advancedScheme.add_group(name='fiscal years 2 and 3', filter_def=filter_fy2and3, targets=all_targets_fy2and3)
# Add targets for the fiscal year groups
fy_targets = {'fiscal year 1': 40, 'fiscal years 2 and 3': 60}
advancedScheme.group_targets(fy_targets)
# Calculate weights
responseweights_adv = dataset.weight(advancedScheme, weight_name='weight', unique_key='responseid',inplace=False)
responseweights_adv = responseweights_adv.reindex(['responseid', 'weight'],axis='columns')
Weighting example - CELL WEIGHT
# create set with only cellweights for advanced scheme (fiscal year groups)
wdf = dataset.weight(advancedScheme, weight_name='weight', unique_key='responseid',inplace=False)
wdf = wdf.drop(columns=['responseid'])
wdfdistinct = wdf.drop_duplicates(subset=['fiscal_year', 'gender', 'age_group', 'region', 'weight'], inplace=False).reindex(['fiscal_year', 'gender', 'age_group', 'region', 'weight'],axis='columns').sort_values(by=['fiscal_year', 'gender', 'age_group', 'region'])
Weighting example - INTERLOCKED
# Create interlocked variable in dataset ('age_group' interlocked with 'gender')
interlocked_dataset = dataset.clone()
interlocked_dataset.interlock("age_gender", "age and gender", ['age_group','gender'], '/')
interlocked_dataset.convert('age_gender', 'single')
# set interlocked cell targets (cells correspond to [age_group 1, gender 1], [age_group 1, gender 2], [age_group 2, gender 1], ...)
interlockedScheme = qp.Rim('test_interlocked')
age_gender_target = {'age_gender': {1: 15, 2: 15, 3:10, 4: 5, 5: 10, 6: 15, 7: 10, 8: 5, 9: 10, 10: 5 }}
interlockedScheme.set_targets(targets=age_gender_target, group_name='interlocked weights')
# Calculate weights
interlocked_dataset.weight(interlockedScheme, weight_name='weight', unique_key='responseid',inplace=True)
interlockedweights = interlocked_dataset.subset(['age_group', 'gender', 'weight'], inplace=False).data().drop_duplicates(subset=['age_group', 'gender'], inplace=False).reindex(['age_group', 'gender', 'weight'],axis='columns').sort_values(by=['age_group'])
Prepare data for writing
# Convert to json
weights_basic = responseweights_basic.to_json(orient="records")
weights_advanced = responseweights_adv.to_json(orient="records")
weights_cell = wdfdistinct.to_json(orient="records")
weights_interlocked = interlockedweights.to_json(orient="records")
Write data to SmartHub custom data
# Insert records for response weights in custom table in hub
headers = {'Authorization': 'Bearer ' + token, "Accept": "application/json", "Content-Type": "application/json"}
# Basic weights
url = site_url + 'v1/hubs/' + hubid +'/tables/'+ hub_tableid_basic +'/records'
response = req.put(url, data = weights_basic, headers = headers)
print(response.status_code)
# Advanced weights
url = site_url + 'v1/hubs/' + hubid +'/tables/'+ hub_tableid_advanced +'/records'
response = req.put(url, data = weights_advanced, headers = headers)
print(response.status_code)
# cell weights
url = site_url + 'v1/hubs/' + hubid +'/tables/'+ hub_tableid_cellweights +'/records'
response = req.put(url, data = weights_cell, headers = headers)
print(response.status_code)
# interlocked weights
url = site_url + 'v1/hubs/' + hubid +'/tables/'+ hub_tableid_interlocked +'/records'
response = req.put(url, data = weights_interlocked, headers = headers)
print(response.status_code)
Prepare data for writing data back to the survey
class Obj(): pass
#Rename column to map survey data variable
weights = responseweights_adv.rename(columns={"weight": "adv_weight"}).head(10)
#Create json for survey weights
payload = Obj()
payload.dataSchema = Obj()
payload.dataSchema.keys = ["responseid"]
payload.dataSchema.variables = ["adv_weight"]
payload.data = weights.to_dict(orient='records')
payload_json = json.dumps(payload, default=lambda o: o.__dict__)
Authenticate
# Get access token
response = req.post(idp_url + 'identity/connect/token',
data = "grant_type=api-user&scope=pub.surveys pub.hubs",
auth=(client_id, client_secret),
headers = {'Content-Type': 'application/x-www-form-urlencoded'})
token = response.json()['access_token']
print(response.status_code)
Write data to the survey
headers = {'Authorization': 'Bearer ' + token, "Accept": "application/json", "Content-Type": "application/json"}
# Advanced weights
response = req.put(site_url + 'v1/surveys/' + projectid + '/responses/data', data = payload_json, headers = headers)
print(response.status_code)
print(response.content)