Qumo/JPMC-quant
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

automatic generation of all plots with all combinations

Browse Source
This commit is contained in:
Luigi Maiorano
2026-02-03 15:03:57 +01:00
parent e44251c3d6
commit 2817ed240a
4 changed files with 135 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
03_quant_report.script.py
View File

@@ -18,27 +18,39 @@ from speaking_styles import SPEAKING_STYLES
# CLI argument parsing for batch automation
# When run as script: python 03_quant_report.script.py --age '["18 to 21 years"]' --consumer '["Starter"]'
# When run in Jupyter: args will use defaults (all filters = None = all options selected)
# Central filter configuration - add new filters here only
# Format: 'cli_arg_name': 'QualtricsSurvey.options_* attribute name'
FILTER_CONFIG = {
'age': 'options_age',
'gender': 'options_gender',
'ethnicity': 'options_ethnicity',
'income': 'options_income',
'consumer': 'options_consumer',
# Add new filters here: 'newfilter': 'options_newfilter',
}
def parse_cli_args():
parser = argparse.ArgumentParser(description='Generate quant report with optional filters')
parser.add_argument('--age', type=str, default=None, help='JSON list of age groups')
parser.add_argument('--gender', type=str, default=None, help='JSON list of genders')
parser.add_argument('--ethnicity', type=str, default=None, help='JSON list of ethnicities')
parser.add_argument('--income', type=str, default=None, help='JSON list of income groups')
parser.add_argument('--consumer', type=str, default=None, help='JSON list of consumer segments')
# Dynamically add filter arguments from config
for filter_name in FILTER_CONFIG:
parser.add_argument(f'--{filter_name}', type=str, default=None, help=f'JSON list of {filter_name} values')
parser.add_argument('--filter-name', type=str, default=None, help='Name for this filter combination (used for .txt description file)')
# Only parse if running as script (not in Jupyter/interactive)
try:
# Check if running in Jupyter by looking for ipykernel
get_ipython() # noqa: F821
return argparse.Namespace(age=None, gender=None, ethnicity=None, income=None, consumer=None)
# Return namespace with all filters set to None
return argparse.Namespace(**{f: None for f in FILTER_CONFIG}, filter_name=None)
except NameError:
args = parser.parse_args()
# Parse JSON strings to lists
args.age = json.loads(args.age) if args.age else None
args.gender = json.loads(args.gender) if args.gender else None
args.ethnicity = json.loads(args.ethnicity) if args.ethnicity else None
args.income = json.loads(args.income) if args.income else None
args.consumer = json.loads(args.consumer) if args.consumer else None
for filter_name in FILTER_CONFIG:
val = getattr(args, filter_name)
setattr(args, filter_name, json.loads(val) if val else None)
return args
cli_args = parse_cli_args()
@@ -100,13 +112,60 @@ BEST_CHOSEN_CHARACTER = "the_coach"
# %%
# mo.stop(filter_form.value is None, mo.md("**Please submit filter above to proceed**"))
# CLI args: None means "all options selected" (use S.options_* defaults)
_filter_age = cli_args.age if cli_args.age is not None else S.options_age
_filter_gender = cli_args.gender if cli_args.gender is not None else S.options_gender
_filter_ethnicity = cli_args.ethnicity if cli_args.ethnicity is not None else S.options_ethnicity
_filter_income = cli_args.income if cli_args.income is not None else S.options_income
_filter_consumer = cli_args.consumer if cli_args.consumer is not None else S.options_consumer
# Build filter values dict dynamically from FILTER_CONFIG
_active_filters = {}
for filter_name, options_attr in FILTER_CONFIG.items():
cli_value = getattr(cli_args, filter_name)
all_options = getattr(S, options_attr)
_active_filters[filter_name] = cli_value if cli_value is not None else all_options
_d = S.filter_data(data_all, age=_filter_age, gender=_filter_gender, income=_filter_income, ethnicity=_filter_ethnicity, consumer=_filter_consumer)
_d = S.filter_data(data_all, **_active_filters)
# Write filter description file if filter-name is provided
if cli_args.filter_name and S.fig_save_dir:
# Get the filter slug (e.g., "All_Respondents", "Cons-Starter", etc.)
_filter_slug = S._get_filter_slug()
_filter_slug_dir = S.fig_save_dir / _filter_slug
_filter_slug_dir.mkdir(parents=True, exist_ok=True)
# Build filter description
_filter_desc_lines = [
f"Filter: {cli_args.filter_name}",
"",
"Applied Filters:",
]
_short_desc_parts = []
for filter_name, options_attr in FILTER_CONFIG.items():
all_options = getattr(S, options_attr)
values = _active_filters[filter_name]
display_name = filter_name.replace('_', ' ').title()
if values != all_options:
_short_desc_parts.append(f"{display_name}: {', '.join(values)}")
_filter_desc_lines.append(f" {display_name}: {', '.join(values)}")
else:
_filter_desc_lines.append(f" {display_name}: All")
# Write detailed description INSIDE the filter-slug directory
_filter_file = _filter_slug_dir / f"{cli_args.filter_name}.txt"
_filter_file.write_text('\n'.join(_filter_desc_lines))
# Append to summary index file at figures/<export_date>/filter_index.txt
_summary_file = S.fig_save_dir / "filter_index.txt"
_short_desc = "; ".join(_short_desc_parts) if _short_desc_parts else "All Respondents"
_summary_line = f"{_filter_slug} | {cli_args.filter_name} | {_short_desc}\n"
# Append or create the summary file
if _summary_file.exists():
_existing = _summary_file.read_text()
# Avoid duplicate entries for same slug
if _filter_slug not in _existing:
with _summary_file.open('a') as f:
f.write(_summary_line)
else:
_header = "Filter Index\n" + "=" * 80 + "\n\n"
_header += "Directory | Filter Name | Description\n"
_header += "-" * 80 + "\n"
_summary_file.write_text(_header + _summary_line)
# Stop execution and prevent other cells from running if no data is selected
# mo.stop(len(_d.collect()) == 0, mo.md("**No Data available for current filter combination**"))

50
README.md
View File

@@ -94,7 +94,7 @@ combinations.append({
})
```
4. **Filter keys** must match CLI argument names:
4. **Filter keys** must match CLI argument names (defined in `FILTER_CONFIG` in `03_quant_report.script.py`):
- `age` — values from `survey.options_age`
- `gender` — values from `survey.options_gender`
- `ethnicity` — values from `survey.options_ethnicity`
@@ -144,4 +144,50 @@ combinations.append({
- **Empty filters dict** = all respondents (no filtering)
- **Omitted filter keys** = all options for that dimension selected
- **Output folder names** are auto-generated from active filters by `QualtricsSurvey.filter_data()`
- **Output folder names** are auto-generated from active filters by `QualtricsSurvey.filter_data()`
---
## Adding a New Filter Dimension
To add an entirely new filter dimension (e.g., a new demographic question), edit **only** `FILTER_CONFIG` in `03_quant_report.script.py`:
### Checklist
1. **Ensure `QualtricsSurvey`** has the corresponding `options_*` attribute and `filter_data()` accepts the parameter
2. **Open** `03_quant_report.script.py`
3. **Find** `FILTER_CONFIG` near the top of the file:
```python
FILTER_CONFIG = {
'age': 'options_age',
'gender': 'options_gender',
'ethnicity': 'options_ethnicity',
'income': 'options_income',
'consumer': 'options_consumer',
# Add new filters here: 'newfilter': 'options_newfilter',
}
```
4. **Add** your new filter:
```python
FILTER_CONFIG = {
'age': 'options_age',
'gender': 'options_gender',
'ethnicity': 'options_ethnicity',
'income': 'options_income',
'consumer': 'options_consumer',
'region': 'options_region', # ← New filter
}
```
This **automatically**:
- Adds `--region` CLI argument
- Includes it in Jupyter mode (defaults to all options)
- Passes it to `S.filter_data()`
- Writes it to the `.txt` filter description file
5. **Update** `run_filter_combinations.py` to generate combinations for the new filter (optional)

11
run_filter_combinations.py
View File

@@ -110,12 +110,13 @@ def get_filter_combinations(survey: QualtricsSurvey) -> list[dict]:
return combinations
def run_report(filters: dict, dry_run: bool = False) -> bool:
def run_report(filters: dict, name: str = None, dry_run: bool = False) -> bool:
"""
Run the report script with given filters.
Args:
filters: Dict of filter_name -> list of values
name: Name for this filter combination (used for .txt description file)
dry_run: If True, just print command without running
Returns:
@@ -123,6 +124,10 @@ def run_report(filters: dict, dry_run: bool = False) -> bool:
"""
cmd = [sys.executable, str(REPORT_SCRIPT)]
# Add filter-name for description file
if name:
cmd.extend(['--filter-name', name])
for filter_name, values in filters.items():
if values:
cmd.extend([f'--{filter_name}', json.dumps(values)])
@@ -166,7 +171,7 @@ def main():
print("\nDRY RUN - Commands that would be executed:")
for combo in combinations:
print(f"\n{combo['name']}:")
run_report(combo['filters'], dry_run=True)
run_report(combo['filters'], name=combo['name'], dry_run=True)
return
# Run each combination with progress bar
@@ -175,7 +180,7 @@ def main():
for combo in tqdm(combinations, desc="Running reports", unit="filter"):
tqdm.write(f"Running: {combo['name']}")
if run_report(combo['filters']):
if run_report(combo['filters'], name=combo['name']):
successful += 1
else:
failed.append(combo['name'])

3
utils.py
View File

@@ -750,6 +750,7 @@ class QualtricsSurvey(QualtricsPlotsMixin):
self.filter_consumer:list = None
self.filter_ethnicity:list = None
self.filter_income:list = None
@@ -887,6 +888,8 @@ class QualtricsSurvey(QualtricsPlotsMixin):
if income is not None:
q = q.filter(pl.col('QID15').is_in(income))
self
self.data_filtered = q
return self.data_filtered