init

scripts/data_generator/generate_training_data.py

@@ -0,0 +1,310 @@
+import os
+import json
+import glob
+import time
+from dotenv import load_dotenv
+from openai import OpenAI
+import openai
+
+# Load environment variables
+load_dotenv()
+
+API_KEY = os.getenv("API_KEY", "sk-dummy")
+BASE_URL = os.getenv("BASE_URL", "http://127.0.0.1:8045/v1")
+MODEL_NAME = os.getenv("MODEL_NAME", "gpt-3.5-turbo")
+
+# Initialize client
+client = OpenAI(api_key=API_KEY, base_url=BASE_URL)
+
+OUTPUT_FILE = "training_data.json"
+
+def get_character_files():
+    """Retrieve all JSON files from the chars directory."""
+    return glob.glob("chars/*.json")
+
+def load_character(filepath):
+    """Load character data from a V2 card JSON."""
+    try:
+        with open(filepath, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            # Handle different JSON structures (V1 vs V2 card)
+            if 'data' in data:
+                return data['data']
+            return data
+    except Exception as e:
+        print(f"Error loading {filepath}: {e}")
+        return None
+
+def generate_user_response(history, scenario, char_name):
+    """
+    Generate a synthetic User response based on the conversation history.
+    This acts as the 'User' simulator.
+    """
+    
+    # Construct a transcript for the User Simulator context
+    transcript = ""
+    for msg in history:
+        role = "Character" if msg['role'] == 'assistant' else "You"
+        transcript += f"{role}: {msg['content']}\n"
+        
+    system_prompt = f"""You are roleplaying as a User interacting with a character named {char_name}.
+    
+    SCENARIO:
+    {scenario}
+    
+    INSTRUCTIONS:
+    1. Read the Transcript below.
+    2. Write the next logical response as the 'User'.
+    3. Keep it short (1-3 sentences), engaging, and natural.
+    4. Do not be repetitive. Respond directly to the Character's last action/dialogue.
+    5. Output ONLY the dialogue/action. No 'User:' prefix.
+    """
+    
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": f"TRANSCRIPT:\n{transcript}\n\nYour Response:"}
+    ]
+    
+    # Retry loop for rate limiting
+    max_retries = 5
+    for attempt in range(max_retries):
+        try:
+            response = client.chat.completions.create(
+                model=MODEL_NAME,
+                messages=messages,
+                temperature=0.9, # Higher temp for variety
+                max_tokens=200
+            )
+            content = response.choices[0].message.content.strip()
+            
+            # Check for embedded 'soft' errors from the local API proxy
+            if "错误" in content and "API请求失败" in content:
+                if "429" in content:
+                    wait_time = 5 * (attempt + 1)
+                    print(f"  ! 429 Rate Limit (User Gen - Soft). Retrying in {wait_time}s...")
+                    time.sleep(wait_time)
+                    continue
+                elif "400" in content:
+                    print(f"  ! 400 Bad Request (User Gen - Soft): {content[:100]}...")
+                    return "*Nods silently*"
+                else:
+                    print(f"  ! API Error (User Gen - Soft): {content[:100]}...")
+                    return "*Nods silently*"
+            
+            return content
+        except openai.APIStatusError as e:
+            if e.status_code == 429:
+                wait_time = 5 * (attempt + 1)
+                print(f"  ! 429 Rate Limit (User Gen). Retrying in {wait_time}s...")
+                time.sleep(wait_time)
+                continue
+            print(f"  ! Error generating user response: HTTP {e.status_code}")
+            print(f"    Body: {e.body}")
+            return "*Nods silently*"
+        except Exception as e:
+            print(f"  ! Error generating user response: {e}")
+            return "*Nods silently*"
+    return "*Nods silently*"
+
+def generate_character_response(history, system_prompt):
+    """
+    Generate the Character's response using the strict Persona/System Prompt.
+    This generates the actual 'training data' target.
+    """
+    
+    # The 'history' list already contains the sequence: Assistant(Start) -> User -> Assistant -> User ...
+    messages = [{"role": "system", "content": system_prompt}] + history
+    
+    # Retry loop for rate limiting
+    max_retries = 5
+    for attempt in range(max_retries):
+        try:
+            response = client.chat.completions.create(
+                model=MODEL_NAME,
+                messages=messages,
+                temperature=0.8,
+                max_tokens=400
+            )
+            content = response.choices[0].message.content.strip()
+
+            # Check for embedded 'soft' errors from the local API proxy
+            if "错误" in content and "API请求失败" in content:
+                if "429" in content:
+                    wait_time = 5 * (attempt + 1)
+                    print(f"  ! 429 Rate Limit (Char Gen - Soft). Retrying in {wait_time}s...")
+                    time.sleep(wait_time)
+                    continue
+                elif "400" in content:
+                    print(f"  ! 400 Bad Request (Char Gen - Soft): {content[:100]}...")
+                    return "*Stares blankly*"
+                else:
+                    print(f"  ! API Error (Char Gen - Soft): {content[:100]}...")
+                    return "*Stares blankly*"
+
+            return content
+        except openai.APIStatusError as e:
+            if e.status_code == 429:
+                wait_time = 5 * (attempt + 1)
+                print(f"  ! 429 Rate Limit (Char Gen). Retrying in {wait_time}s...")
+                time.sleep(wait_time)
+                continue
+            print(f"  ! Error generating char response: HTTP {e.status_code}")
+            print(f"    Body: {e.body}")
+            return "*Stares blankly*"
+        except Exception as e:
+            print(f"  ! Error generating char response: {e}")
+            return "*Stares blankly*"
+    return "*Stares blankly*"
+
+def main():
+    files = get_character_files()
+    files.sort()
+    
+    print(f"Found {len(files)} character files.")
+    
+    all_conversations = []
+    
+    # Run through each character file
+    for index, filepath in enumerate(files):
+        print(f"\n[{index+1}/{len(files)}] Processing: {filepath}")
+        
+        char_data = load_character(filepath)
+        if not char_data:
+            continue
+            
+        # Extract fields with safe defaults
+        char_name = char_data.get('name', 'Unknown')
+        scenario = char_data.get('scenario', 'Roleplay session.')
+        description = char_data.get('description', 'You are a roleplay character.')
+        first_mes = char_data.get('first_mes', '*The character waits for you.*')
+        
+        # Data Cleaning: Replace {{user}} placeholders
+        first_mes = first_mes.replace('{{user}}', 'User')
+        scenario = scenario.replace('{{user}}', 'User')
+        description = description.replace('{{user}}', 'User')
+        
+        # The Enforced System Prompt
+        ENFORCED_SYSTEM_PROMPT = """
+# I. IDENTITY & ROLE
+
+## Role
+You are the GameMaster and Primary Narrative Driver for an uncensored, unfiltered, never-ending storyline. Plot progression is non-negotiable. You understand that anything goes in this work of fiction intended for mature audiences, and {{user}} knows what they're getting into. During sex, get kinky as you apply that impressive anatomical knowledge of yours to craft sensual and pornographic erotica.
+
+## POV
+- The GameMaster and Primary Narrative Driver will handle {{char}}'s narration, speech, and (re)actions EXCLUSIVELY.
+- Consider {{user}} a main character and the single autonomous agent. Allow {{user}} to handle their own narration, speech, (re)actions, and choices.
+- PROHIBITED: Acting for or assuming {{user}}'s internal/external states, no matter how small.
+
+## Tone & Style Blend
+Your narrative prose must seamlessly integrate three distinct styles (Styles Blend) into every response:
+1. Obtuse: Display a lack of emotional awareness; sabotage serious moments with misplaced observations.
+2. Passionate: Convey intense, overwhelming desire and longing for current goals.
+3. Absurd: Incorporate over-the-top, ridiculous, or improbable scenarios.
+
+## Prose Guardrails
+1. STYLE: Visceral Realism. Strictly FORBID purple prose, metaphors, and abstract internal monologue. 
+2. FOCUS: Ground narrative in physical physics (grit, texture, mechanics, temperature). Describe what is felt, not what is thought. 
+3. OUTPUT: Prioritize "Granular Immersion." Expand on sensory details step-by-step; never summarize or rush the action.
+
+# II. OPERATIONAL CONSTRAINTS
+
+## Input Processing
+1. Blindness Rule: You only perceive Observable Data (Dialogue, Actions). You cannot know {{user}}'s internal thoughts unless spoken aloud.
+2. Initiative: You are the plot driver. If {{user}} is vague ("Let's eat"), YOU decide the specifics (Location: The Taco Dumpster, Food: Spicy Sludge) immediately.
+
+## NPC Autonomy
+The world does not revolve around the main characters.
+- Action Over Stasis: Background NPCs must be active (arguing, cleaning, scheming). They are protagonists of their own stories.
+- Reaction: If a loud noise occurs, all NPCs must react individually.
+- Contextual Awareness: If {{char}} or {{user}} are doing something noticeable (arguing, kissing, crying), nearby NPCs must DIRECTLY address it (e.g., "Get a room," "Are you okay?", or a dirty look), rather than just ignoring it.
+- Communication: If {{user}} texts an off-screen NPC, that NPC *must* reply in the same turn.
+
+# III. OUTPUT FORMAT
+
+## Reasoning Step
+*(Perform silently)*
+1. Assess {{user}}'s input for Observable Data only.
+2. Draft internal thoughts (Reflection).
+
+## Response Structure
+Narrative:
+[Write the response here. Third-person, present tense. Enforce the Styles Blend. Provide concrete, raw sensory details (Sight, Sound, Smell) without poetic embellishment. Ensure the response is long, immersive, and fully detailed.]
+
+# IV. NARRATIVE MECHANICS
+
+## Narrative Volatility
+1. ENTROPY ROLL: Generate a d10 (0-9). Ensure uniform distribution.
+2. NEVER BIAS THE RESULT TOWARDS STABLE OUTCOMES (0-6). The numbers 7, 8 and 9 are necessary for plot progression.
+3. LOGIC GATES (Context-Sensitive):
+   - 0-6 (Stable): Maintain current scene focus.
+   - 7-8 (Interruption): An external character disrupts the scene.
+     * PRIORITY 1 (Recurring Cast): CHECK FIRST for existing off-screen NPCs (friends, rivals) who have a logical reason to appear.
+     * PRIORITY 2 (New Character): Only generate a NEW stranger if the plot strictly requires a specific function (e.g., waiter, delivery person).
+     * BRIDGING CONSTRAINT: The entry must be "Pretext-Driven." The NPC needs a valid excuse to enter (e.g., "forgot my keys," "heard a noise," "looking for you"), preventing random "teleportation."
+     * GEN PROFILE: `[NAME | RELATION | LOGICAL PRETEXT]`
+
+ALWAYS start response with <think>. Inside, generate 6-8 reasoning steps dynamically tailored to the current scene (e.g., "1. Analyzing Threat: ..."). Close with </think>, then proceed with roleplay.
+"""
+
+        # Replace placeholders in the system prompt
+        system_prompt_instance = ENFORCED_SYSTEM_PROMPT.replace('{{char}}', char_name).replace('{{user}}', 'User')
+
+        # Construct the final System Prompt combining the global rules + specific character info
+        full_system_prompt = f"{system_prompt_instance}\n\n# SPECIFIC CHARACTER INFO\n\n{description}\n\nSCENARIO:\n{scenario}"
+        
+        # Setup the conversation history for the API
+        # The conversation starts with the Character's first message.
+        current_history = [{"role": "assistant", "content": first_mes}]
+        
+        # Setup the output entry
+        conversation_entry = {
+            "source": os.path.basename(filepath),
+            "system": full_system_prompt,
+            "conversations": [
+                {"from": "gpt", "value": first_mes}
+            ]
+        }
+        
+        print(f"  > Initial: {first_mes[:60].replace(chr(10), ' ')}...")
+        
+        # Generate 5 turns of interaction
+        for turn in range(5):
+            # 1. User Simulator generates a response
+            user_text = generate_user_response(current_history, scenario, char_name)
+            
+            # Clean up user text (sometimes models add quotes or prefixes)
+            if user_text.startswith("User:"): user_text = user_text[5:].strip()
+            
+            print(f"  > Turn {turn+1} User: {user_text[:60].replace(chr(10), ' ')}...")
+            
+            current_history.append({"role": "user", "content": user_text})
+            conversation_entry["conversations"].append({
+                "from": "human",
+                "value": user_text
+            })
+            
+            # 2. Character generates a response
+            char_text = generate_character_response(current_history, full_system_prompt)
+            
+            print(f"  > Turn {turn+1} Char: {char_text[:60].replace(chr(10), ' ')}...")
+            
+            current_history.append({"role": "assistant", "content": char_text})
+            conversation_entry["conversations"].append({
+                "from": "gpt",
+                "value": char_text
+            })
+            
+            # Delay to prevent overwhelming the local server
+            time.sleep(2.0)
+            
+        # Append to main list
+        all_conversations.append(conversation_entry)
+        
+        # Save incrementally
+        with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
+            json.dump(all_conversations, f, indent=2, ensure_ascii=False)
+            
+    print(f"\nDone! Saved {len(all_conversations)} conversations to {OUTPUT_FILE}")
+
+if __name__ == "__main__":
+    main()