File I/O & Environment Variables

AI applications are configuration-heavy. You have API keys, model names, temperature settings, prompt template files, output directories, and environment-specific endpoints. Knowing how to handle files and environment configuration cleanly is foundational — it determines whether your app is secure, portable, and easy to run in new environments.

pathlib: The Modern Way to Handle Paths

If you've used C#'s System.IO.Path or FileInfo, you know how awkward string-based path manipulation gets. Python's pathlib module (introduced in 3.4) gives you an object-oriented path API that's cleaner, cross-platform, and composable with / as the path separator operator.

string root = AppDomain.CurrentDomain.BaseDirectory;
                      string promptDir = Path.Combine(root, "prompts");
                      string file = Path.Combine(promptDir, "system.txt");
                      
                      bool exists = File.Exists(file);
                      string content = File.ReadAllText(file);

from pathlib import Path
                      
                      root = Path(__file__).parent  # directory of this .py file
                      prompt_dir = root / "prompts"  # / operator joins paths
                      file = prompt_dir / "system.txt"
                      
                      exists = file.exists()
                      content = file.read_text(encoding="utf-8")

Essential pathlib Operations for AI Projects

from pathlib import Path
                      
                      # Project root detection
                      PROJECT_ROOT = Path(__file__).resolve().parent.parent
                      
                      # Common directories
                      PROMPTS_DIR = PROJECT_ROOT / "prompts"
                      DATA_DIR = PROJECT_ROOT / "data"
                      OUTPUT_DIR = PROJECT_ROOT / "outputs"
                      
                      # Create directories if missing
                      OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
                      
                      # List all .txt prompt files
                      prompt_files = list(PROMPTS_DIR.glob("*.txt"))
                      
                      # Read a prompt template
                      system_prompt = (PROMPTS_DIR / "system.txt").read_text(encoding="utf-8")
                      
                      # Write an AI response to disk
                      output_file = OUTPUT_DIR / "response_2025_06_07.txt"
                      output_file.write_text(response_text, encoding="utf-8")
                      
                      # Path introspection
                      print(output_file.name)       # "response_2025_06_07.txt"
                      print(output_file.stem)       # "response_2025_06_07"
                      print(output_file.suffix)     # ".txt"
                      print(output_file.parent)     # outputs directory path

Reading and Writing Files

Text Files: Prompt Templates and Logs

from pathlib import Path
                      from datetime import datetime
                      
                      def load_prompt_template(template_name: str) -> str:
                          """Load a prompt template from the prompts directory."""
                          template_path = PROMPTS_DIR / f"{template_name}.txt"
                          if not template_path.exists():
                              raise FileNotFoundError(f"Prompt template not found: {template_path}")
                          return template_path.read_text(encoding="utf-8")
                      
                      
                      def save_response(prompt: str, response: str, tag: str = "") -> Path:
                          """Save a prompt-response pair to disk for review."""
                          timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
                          filename = f"response_{timestamp}{'_' + tag if tag else ''}.txt"
                          output = OUTPUT_DIR / filename
                      
                          content = f"=== PROMPT ===\n{prompt}\n\n=== RESPONSE ===\n{response}\n"
                          output.write_text(content, encoding="utf-8")
                          return output
                      
                      
                      # Using open() for streaming or append mode:
                      def append_to_log(message: str, log_file: Path) -> None:
                          with open(log_file, "a", encoding="utf-8") as f:
                              timestamp = datetime.now().isoformat()
                              f.write(f"[{timestamp}] {message}\n")

JSON Files: Saving Structured AI Outputs

import json
                      from pathlib import Path
                      
                      def save_evaluation_results(results: list[dict], output_path: Path) -> None:
                          """Save structured evaluation data as pretty-printed JSON."""
                          with open(output_path, "w", encoding="utf-8") as f:
                              json.dump(results, f, indent=2, ensure_ascii=False)
                      
                      
                      def load_evaluation_results(path: Path) -> list[dict]:
                          """Load evaluation results from JSON."""
                          if not path.exists():
                              return []
                          return json.loads(path.read_text(encoding="utf-8"))
                      
                      
                      # Example usage
                      results = [
                          {"prompt_id": "p001", "model": "claude-3-5-sonnet", "score": 0.92, "latency_ms": 1240},
                          {"prompt_id": "p002", "model": "claude-3-5-sonnet", "score": 0.85, "latency_ms": 980},
                      ]
                      save_evaluation_results(results, OUTPUT_DIR / "eval_results.json")

Environment Variables and Configuration

AI applications need to run in at least three contexts: local development, CI/CD, and production. Configuration that works in all three uses environment variables — not hardcoded values, not config files with secrets baked in.

The .env Pattern

# .env — local development secrets (NEVER commit to git)
                      ANTHROPIC_API_KEY=sk-ant-api03-xxxx
                      OPENAI_API_KEY=sk-xxxx
                      PINECONE_API_KEY=xxxx
                      APP_ENV=development
                      MODEL_NAME=claude-3-5-sonnet-20241022
                      MAX_TOKENS=1024
                      LOG_LEVEL=DEBUG

# .env.example — committed to git as a template
                      ANTHROPIC_API_KEY=your_key_here
                      OPENAI_API_KEY=your_key_here
                      PINECONE_API_KEY=your_key_here
                      APP_ENV=development
                      MODEL_NAME=claude-3-5-sonnet-20241022
                      MAX_TOKENS=1024
                      LOG_LEVEL=INFO

# src/config.py — centralized configuration loader
                      from __future__ import annotations
                      import os
                      from pathlib import Path
                      from dotenv import load_dotenv
                      
                      # Load .env file — does nothing if already loaded or running in CI with real env vars
                      load_dotenv()
                      
                      
                      def require_env(key: str) -> str:
                          """Get an environment variable or raise a clear error."""
                          value = os.environ.get(key)
                          if not value:
                              raise EnvironmentError(
                                  f"Required environment variable '{key}' is not set. "
                                  f"Add it to your .env file or set it in your environment."
                              )
                          return value
                      
                      
                      def optional_env(key: str, default: str) -> str:
                          """Get an optional environment variable with a fallback."""
                          return os.environ.get(key, default)
                      
                      
                      # Application configuration as module-level constants
                      ANTHROPIC_API_KEY = require_env("ANTHROPIC_API_KEY")
                      APP_ENV = optional_env("APP_ENV", "development")
                      MODEL_NAME = optional_env("MODEL_NAME", "claude-3-5-sonnet-20241022")
                      MAX_TOKENS = int(optional_env("MAX_TOKENS", "1024"))
                      LOG_LEVEL = optional_env("LOG_LEVEL", "INFO")

Using Configuration in Your Application

# main.py
                      import anthropic
                      from src.config import ANTHROPIC_API_KEY, MODEL_NAME, MAX_TOKENS
                      
                      # Use the config — no env access scattered throughout codebase
                      client = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY)
                      
                      response = client.messages.create(
                          model=MODEL_NAME,
                          max_tokens=MAX_TOKENS,
                          messages=[{"role": "user", "content": "Hello!"}],
                      )

Environment-Specific Configuration

Production AI applications often need different settings per environment. A clean pattern uses an APP_ENV variable to switch behavior without code changes:

# src/config.py (extended)
                      import os
                      from dataclasses import dataclass
                      from dotenv import load_dotenv
                      
                      load_dotenv()
                      
                      @dataclass(frozen=True)
                      class AppConfig:
                          anthropic_api_key: str
                          model_name: str
                          max_tokens: int
                          log_level: str
                          is_production: bool
                          output_dir: str
                      
                      
                      def load_config() -> AppConfig:
                          env = os.environ.get("APP_ENV", "development")
                          is_production = env == "production"
                      
                          return AppConfig(
                              anthropic_api_key=os.environ["ANTHROPIC_API_KEY"],
                              model_name=os.environ.get(
                                  "MODEL_NAME",
                                  "claude-3-5-sonnet-20241022" if is_production else "claude-3-haiku-20240307",
                              ),
                              max_tokens=int(os.environ.get("MAX_TOKENS", "1024")),
                              log_level=os.environ.get("LOG_LEVEL", "WARNING" if is_production else "DEBUG"),
                              is_production=is_production,
                              output_dir=os.environ.get("OUTPUT_DIR", "./outputs"),
                          )
                      
                      
                      # Singleton config loaded once at startup
                      config = load_config()

Processing Prompt Template Files

Storing prompt templates in files instead of hardcoding them in Python strings gives you several benefits: version-controlled prompts, non-programmer access to prompt engineering, and easy A/B testing of prompt variants.

# prompts/code_review.txt
                      """
                      You are a senior Python engineer performing a code review.
                      Focus on: {focus_areas}
                      
                      Review the following code for issues related to correctness,
                      performance, security, and readability. Be specific and cite
                      line references where possible.
                      
                      Code language: {language}
                      Context: {context}
                      """
                      
                      # src/prompts.py
                      from pathlib import Path
                      from typing import Any
                      
                      PROMPTS_DIR = Path(__file__).parent.parent / "prompts"
                      
                      
                      def load_template(name: str, **variables: Any) -> str:
                          """
                          Load a prompt template and fill in variables.
                          Variables use Python's .format() syntax: {variable_name}
                          """
                          template_path = PROMPTS_DIR / f"{name}.txt"
                          raw_template = template_path.read_text(encoding="utf-8").strip()
                      
                          if variables:
                              try:
                                  return raw_template.format(**variables)
                              except KeyError as e:
                                  raise ValueError(f"Template '{name}' is missing variable: {e}")
                      
                          return raw_template
                      
                      
                      # Usage
                      code_review_prompt = load_template(
                          "code_review",
                          focus_areas="security, error handling",
                          language="Python",
                          context="Production AI pipeline processing 10k req/day",
                      )
                      
                      print(code_review_prompt[:100])