BF667-AI commited on
Commit
713d2ba
·
verified ·
1 Parent(s): 12fc364

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +60 -527
app.py CHANGED
@@ -392,127 +392,66 @@ class StreamProcessor:
392
  # Handle thinking tags
393
  if not in_thought and '<think>' in text:
394
  in_thought = True
395
- history.append({'role': 'assthought = True
396
- history.append({'role': 'assistant', 'contentistant', 'content': '', 'metadata': {'title': '💭 Thought'}})
397
- assistant_message_started': '', 'metadata': {'title': '💭 Thought' = True
398
- after = text.split('<think>', 1)[1]
399
- thought_buf += after}})
400
  assistant_message_started = True
401
- after = text.split('<think>', 1)[1
402
-
403
- if '</think>' in thought_buf:
404
- before,]
405
  thought_buf += after
406
 
407
  if '</think>' in thought_buf:
408
- after2 = thought_buf.split('</think>', 1)
409
  before, after2 = thought_buf.split('</think>', 1)
410
- history[-1]['content'] history[-1]['content'] = before.strip()
411
- in_thought = False
412
- answer_buf = = before.strip()
413
  in_thought = False
414
  answer_buf = after2
415
- history.append({'role': ' after2
416
- history.append({'roleassistant', 'content': answer_buf})
417
- else:
418
- history[-1]['': 'assistant', 'content': answer_buf})
419
  else:
420
- content'] = thought_buf
421
- yield history, ""
422
- continue
423
-
424
- if in_thought:
425
- thought_b history[-1]['content'] = thought_buf
426
  yield history, ""
427
  continue
428
 
429
  if in_thought:
430
  thought_buf += text
431
  if '</think>' in thought_buf:
432
- beforeuf += text
433
- if '</think>' in thought_buf:
434
- , after2 = thought_buf.split('</think>', 1)
435
- before, after2 = thought_buf.split('</think>', 1)
436
  history[-1]['content'] = before.strip()
437
- in_thought = history[-1]['content'] = before.strip()
438
- in_thought False
439
- answer_buf = after2
440
- history.append({'role': = False
441
  answer_buf = after2
442
- history 'assistant', 'content': answer_buf})
443
- else:
444
  history.append({'role': 'assistant', 'content': answer_buf})
445
  else:
446
- history[-1]['content[-1]['content'] = thought_buf
447
- yield history, ""
448
- '] = thought_buf
449
  yield history, ""
450
  continue
451
 
452
  # Stream answer
453
  if not assistant_message_started:
454
- history.append({'role continue
455
-
456
- # Stream answer
457
- if not assistant_message_started:
458
- ': 'assistant', 'content': ''})
459
- assistant_message_start history.append({'role': 'assistant', 'content': ''})
460
  assistant_message_started = True
461
 
462
- answered = True
463
-
464
  answer_buf += text
465
  history[-1]['content'] = answer_buf.strip()
466
- yield history, "_buf += text
467
- history[-1]['content'] = answer_buf.strip()
468
  yield history, ""
469
 
470
- # Main chat function
471
- def chat_response(
472
- user_msg: str,
473
- chat"
474
-
475
  # Main chat function
476
  def chat_response(
477
  user_msg: str,
478
  chat_history: List[Dict],
479
  system_prompt: str,
480
- _history: List[Dict],
481
- system_prompt: str,
482
- enable_search enable_search: bool,
483
- max_results: int,
484
- max_ch: bool,
485
  max_results: int,
486
  max_chars: int,
487
- ars: int,
488
  model_name: str,
489
  max_tokens: int,
490
  temperature: float,
491
- top model_name: str,
492
- max_tokens: int,
493
- _k: int,
494
- top_p temperature: float,
495
  top_k: int,
496
  top_p: float,
497
- repeat: float,
498
  repeat_penalty: float,
499
  search_timeout: float
500
- )_penalty: float,
501
- search_timeout: float
502
  ) -> Generator[Tuple[List[Dict], str], None, None]:
503
- """Generate -> Generator[Tuple[List[Dict], str], None, None]:
504
  """Generate streaming chat responses with search integration"""
505
 
506
- cancel streaming chat responses with search integration"""
507
-
508
  cancel_event.clear()
509
  history = list(chat_history or [])
510
  history.append({'role': 'user', 'content': user_msg})
511
 
512
- #_event.clear()
513
- history = list(chat_history or [])
514
- history.append({'role': 'user', 'content': user_msg})
515
-
516
  # Perform search if enabled
517
  search_results: List[SearchResult] = []
518
  search_debug = "Web search disabled."
@@ -523,44 +462,18 @@ ars: int,
523
  search_results = SearchManager.search(
524
  user_msg,
525
  int(max_results),
526
- Perform search if enabled
527
- search_results: List[SearchResult] = []
528
- search_debug = "Web search disabled."
529
-
530
- if enable_search:
531
- search_debug = "🔍 Searching across multiple engines..."
532
- try:
533
- search_results = SearchManager.search(
534
- user_msg,
535
- int(max int(max_chars),
536
- float(search_timeout)
537
- )
538
-
539
- if search_results:
540
- _results),
541
  int(max_chars),
542
  float(search_timeout)
543
  )
544
 
545
  if search_results:
546
- search_debug = f"✅ Search completed - Found {len(search_results)} results\n\n" + "\n".join search_debug = f"✅ Search completed - Found {len(search_results)} results\n\n" + "\n".join(
547
- f"- {r.format(int(max_chars))}" for r in(
548
  f"- {r.format(int(max_chars))}" for r in search_results
549
  )
550
  else:
551
- search_debug = "❌ No search search_results
552
- )
553
- else:
554
- search_debug = "❌ No search results found. results found. Check internet connection or try again."
555
  except Exception as e:
556
- search_debug = f"❌ Check internet connection or try again."
557
- except Exception as e:
558
- search Search failed: {str(e)}"
559
- logger.error(f"Search error: {e}")
560
-
561
- try:
562
- # Build prompt
563
- if enable_debug = f"❌ Search failed: {str(e)}"
564
  logger.error(f"Search error: {e}")
565
 
566
  try:
@@ -568,53 +481,28 @@ _results),
568
  if enable_search and search_results:
569
  enriched_prompt = PromptBuilder.build_search_context(
570
  search_results,
571
- _search and search_results:
572
- enriched_prompt = PromptBuilder.build_search_context(
573
- search_results,
574
- system_prompt, system_prompt,
575
  user_msg
576
  )
577
  else:
578
- enriched_prompt = system_prompt
579
- user_msg
580
- )
581
- else:
582
- enriched_prompt = system.strip()
583
-
584
- # Load model
585
- pipe = ModelManager.load_pipeline(model_name)
586
-
587
- # Format prompt
588
- prompt = PromptBuilder.format_conversation_prompt.strip()
589
 
590
  # Load model
591
  pipe = ModelManager.load_pipeline(model_name)
592
 
593
  # Format prompt
594
- (history, enriched_prompt, pipe.tokenizer)
595
- prompt_debug = f"\n\n--- Prompt prompt = PromptBuilder.format_conversation(history, enriched_prompt, pipe.tokenizer)
596
- prompt_debug = f"\n\n--- Prompt Preview ---\n```\n{prompt[:500]}...\n``` Preview ---\n```\n{prompt[:500]}...\n" if len(prompt) > 500 else f"\n```" if len(prompt) > 500 else f"\n\n--- Prompt Preview\n--- Prompt Preview ---\n```\n{prompt}\n```"
597
 
598
  # Configure generation
599
  config = GenerationConfig(
600
- max_tokens ---\n```\n{prompt}\n```"
601
-
602
- # Configure generation
603
- config = GenerationConfig(
604
- max_tokens=max_t=max_tokens,
605
- temperature=temperatureokens,
606
  temperature=temperature,
607
  top_k=top_k,
608
  top_p=top_p,
609
  repetition_penalty=repeat_penalty
610
  )
611
 
612
- # Setup,
613
- top_k=top_k,
614
- top_p=top_p,
615
- repetition_penalty=repeat_penalty
616
- )
617
-
618
  # Setup streamer
619
  streamer = TextIteratorStreamer(
620
  pipe.tokenizer,
@@ -622,212 +510,105 @@ _results),
622
  skip_special_tokens=True
623
  )
624
 
625
- # Start streamer
626
- streamer = TextIteratorStreamer(
627
- pipe.tokenizer,
628
- skip_prompt=True,
629
- skip_special_tokens=True
630
- )
631
-
632
- # Start generation in generation in background thread
633
  gen_kwargs = config.to_dict()
634
- gen_kwargs['stream background thread
635
- gen_kwargs = config.to_dict()
636
- gener'] = streamer
637
- gen_kwargs['return_full_text_kwargs['streamer'] = streamer
638
  gen_kwargs['return_full_text'] = False
639
 
640
- '] = False
641
-
642
  gen_thread = threading.Thread(
643
- target= gen_thread = threading.Thread(
644
  target=pipe,
645
  args=(prompt,),
646
  kwargs=gen_kwargs
647
  )
648
  gen_thread.start()
649
 
650
- # Yield initialpipe,
651
- args=(prompt,),
652
- kwargs=gen_kwargs
653
- )
654
- gen_thread.start()
655
-
656
- # Yield initial state state
657
- yield history, search_debug
658
-
659
- # Process stream
660
- for
661
  yield history, search_debug
662
 
663
  # Process stream
664
  for history_update, debug_update in StreamProcessor.process_stream(streamer, history):
665
- yield history_update history_update, debug_update in StreamProcessor.process_stream(streamer, history):
666
  yield history_update, debug_update
667
 
668
  # Wait for completion
669
- , debug_update
670
-
671
- # Wait for completion
672
  gen_thread.join(timeout=5.0)
673
  yield history, search_debug + prompt_debug
674
 
675
- gen_thread.join(timeout=5.0)
676
- yield history, search_debug + prompt_debug
677
-
678
  except GeneratorExit:
679
- logger.info("Generation except GeneratorExit:
680
  logger.info("Generation cancelled by user")
681
  return
682
- except Exception as e:
683
- logger.error(f" cancelled by user")
684
- return
685
  except Exception as e:
686
  logger.error(f"Generation error: {e}")
687
- history.append({'role': 'assistant', 'content': f"Error: {strGeneration error: {e}")
688
- history.append({'role': 'assistant', 'content': f"(e)}"})
689
  yield history, search_debug
690
  finally:
691
  gc.collect()
692
 
693
  # Utility functions
694
  def get_model_size(model_name: str) -> float:
695
- """Error: {str(e)}"})
696
- yield history, search_debug
697
- finally:
698
- gc.collect()
699
-
700
- # Utility functions
701
- def get_model_size(model_name: str) ->Get model size in billions of parameters"""
702
- return MODELS.get(model_name, {}).get("params float:
703
  """Get model size in billions of parameters"""
704
- return MODELS.get(model_name, {}).get("params_b", 4_b", 4.0)
705
-
706
- def get_duration_estimate(
707
- model_name: str,
708
- enable_search: bool,
709
- max_t.0)
710
 
711
  def get_duration_estimate(
712
  model_name: str,
713
  enable_search: bool,
714
  max_tokens: int,
715
  search_timeout: float
716
- ) -> float:
717
- """Calculateokens: int,
718
- search_timeout: float
719
  ) -> float:
720
  """Calculate estimated GPU duration"""
721
  model_size = get_model_size(model_name)
722
- estimated GPU duration"""
723
- model_size = get_model_size(model_name)
724
  use_aot = model_size >= 2
725
 
726
- base use_aot = model_size >= 2
727
-
728
- _duration = 20 if not use_aot else 40
729
- token_duration = max_tokens * base_duration = 20 if not use_aot else 40
730
  token_duration = max_tokens * 0.005
731
  search_duration = 10 if enable_search else 0
732
- a0.005
733
- search_duration = 10 if enable_search else 0
734
  aot_compilation = 20 if use_aot else 0
735
 
736
- return base_dot_compilation = 20 if use_aot else 0
737
-
738
  return base_duration + token_duration + search_duration + aot_compilation
739
 
740
- def update_duration_estimate(
741
- modeluration + token_duration + search_duration + aot_compilation
742
-
743
  def update_duration_estimate(
744
  model_name: str,
745
- enable_name: str,
746
  enable_search: bool,
747
  max_results: int,
748
- max_chars: int_search: bool,
749
- max_results: int,
750
  max_chars: int,
751
  max_tokens: int,
752
  search_timeout: float
753
  ) -> str:
754
  """Format duration estimate for display"""
755
  try:
756
- duration = get_duration_estimate(model_name, enable_search, max_tokens,
757
- max_tokens: int,
758
- search_timeout: float
759
- ) -> str:
760
- """Format duration estimate for display"""
761
- try:
762
- duration = get_duration_estimate(model_name, enable_search, max, search_timeout)
763
  model_size = get_model_size(model_name)
764
 
765
- return f"""⏱_tokens, search_timeout)
766
- model_size = get_model_size(model_name)
767
-
768
- return f️ **Estimated GPU Time: {duration:.1f} seconds**
769
 
770
  📊 **Model Size:** {model_size:.1f}B parameters
771
- 🔍"""⏱️ **Estimated GPU Time: {duration:.1f} seconds**
772
-
773
- 📊 **Model Size:** {model_size:.1f}B parameters
774
- 🔍 ** **Web Search:** {'Enabled (Multi-Engine)' if enable_search else 'Disabled'}"""
775
- except ExceptionWeb Search:** {'Enabled (Multi-Engine)' if enable_search else 'Disabled'}"""
776
  except Exception as e:
777
- logger.error(f"Error calculating estimate: {e}")
778
- return f"⚠️ Error as e:
779
  logger.error(f"Error calculating estimate: {e}")
780
  return f"⚠️ Error calculating estimate: {e}"
781
 
782
  def update_default_prompt(enable_search: bool) -> str:
783
- """Generate calculating estimate: {e}"
784
-
785
- def update_default_prompt(enable_search: bool) -> str:
786
- """ default system prompt"""
787
- return "You are aGenerate default system prompt"""
788
  return "You are a helpful assistant."
789
 
790
  # ------------------------------
791
- # Gradio UI (unchanged)
792
- # ------------------------------
793
- with gr helpful assistant."
794
-
795
- # ------------------------------
796
- # Gradio UI (unchanged)
797
  # ------------------------------
798
  with gr.Blocks(
799
- .Blocks(
800
  title="LLM Inference",
801
- theme=gr.themes.Soft(
802
- primary_hue="blue",
803
- secondary_hue="blue",
804
- neutral_hue="slate",
805
- radius title="LLM Inference",
806
  theme=gr.themes.Soft(
807
  primary_hue="blue",
808
  secondary_hue="blue",
809
  neutral_hue="slate",
810
  radius_size="lg",
811
- font=[gr_size="lg",
812
  font=[gr.themes.GoogleFont("Syne"), "Arial", "sans-serif"]
813
  ),
814
  css="""
815
- .themes.GoogleFont("Syne"), "Arial", "sans-serif"]
816
- ),
817
- css="""
818
- .duration-estimate { background: linear-gradient(135deg, #667eea15 .duration-estimate { background: linear-gradient(135deg, #667eea15 0%, #0%, #764ba215 100%); border-left: 4764ba215 px solid #667eea; padding: 12px; border-radius:100%); border-left: 4px solid #667eea; padding: 12px; border-radius: 8px; margin: 16px 0; }
819
- . 8px; margin: 16px 0; }
820
- .chatbot { border-radius: 12px; box-shadow: 0 4px 6px -chatbot { border-radius: 12px; box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1); }
821
- button.primary {1px rgba(0, 0, 0, 0.1); }
822
- button.primary { font font-weight: 600; }
823
- .gradio-accordion { margin-bottom: 12px; }
824
- """
825
- -weight: 600; }
826
  .gradio-accordion { margin-bottom: 12px; }
827
  """
828
  ) as demo:
829
- # Header
830
- gr.Markdown("") as demo:
831
  # Header
832
  gr.Markdown("""
833
  # 🧠 LLM Inference with Multi-Engine Search
@@ -835,138 +616,67 @@ with gr.Blocks(
835
 
836
  with gr.Row():
837
  # Left Panel - Configuration
838
- with gr.Column(scale"
839
- # 🧠 LLM Inference with Multi-Engine Search
840
- """)
841
-
842
- with gr.Row():
843
- # Left Panel - Configuration
844
- with gr.Column(=3):
845
- # Core Settings (Always Visible)
846
- with gr.Group():
847
- gr.Markdown("scale=3):
848
  # Core Settings (Always Visible)
849
  with gr.Group():
850
- gr.Markdown### ⚙️ Core Settings")
851
- model_dd = gr.D("### ⚙️ Core Settings")
852
- model_dd = grropdown(
853
- label="🤖 Model",
854
- choices=list(MODELS.keys()),
855
- value="Qwen3.Dropdown(
856
  label="🤖 Model",
857
  choices=list(MODELS.keys()),
858
  value="Qwen3-1.7B",
859
  info="Select the language model to use"
860
  )
861
- search_chk = gr.Check-1.7B",
862
- info="Select the language model to use"
863
- )
864
  search_chk = gr.Checkbox(
865
  label="🔍 Enable Web Search",
866
  value=False,
867
- info="Search across Googlebox(
868
- label="🔍 Enable Web Search",
869
- value=False,
870
- info="Search across Google,, DuckDuckGo, and Bing (no API required)"
871
- )
872
- sys_prompt = gr.Textbox(label="📝 System Prompt", lines=3, value=update_default_prompt(False), placeholder="Define the assistant's behavior and personality...")
873
-
874
- # DuckDuckGo, and Bing (no API required)"
875
  )
876
  sys_prompt = gr.Textbox(label="📝 System Prompt", lines=3, value=update_default_prompt(False), placeholder="Define the assistant's behavior and personality...")
877
 
878
- Duration Estimate
879
  duration_display = gr.Markdown(
880
- value=update_duration_estimate("Qwen3-1. # Duration Estimate
881
- duration_display = gr.Markdown(
882
- value=update_duration_estimate("Qwen3-1.7B", False7B", False, 4, 50, 1024, 5.0),
883
- elem_classes="duration-est, 4, 50, 1024, 5.0),
884
  elem_classes="duration-estimate"
885
- imate"
886
  )
887
 
888
  # Advanced Settings (Collapsible)
889
  with gr.Accordion("🎛️ Advanced Generation Parameters", open=False):
890
- )
891
-
892
- # Advanced Settings (Collapsible)
893
- with gr.Accordion("🎛️ Advanced Generation max_tok = gr.Slider(
894
- 64, 16384, value=1024, step=32,
895
- label="Max Tok Parameters", open=False):
896
  max_tok = gr.Slider(
897
  64, 16384, value=1024, step=32,
898
  label="Max Tokens",
899
- infoens",
900
  info="Maximum length of generated response"
901
  )
902
- temp = gr.Slider(
903
- 0.1, ="Maximum length of generated response"
904
- )
905
  temp = gr.Slider(
906
  0.1, 2.0, value=0.7, step=0.1,
907
- label="Temperature2.0, value=0.7, step=0.1,
908
  label="Temperature",
909
  info="Higher = more creative, Lower = more focused"
910
  )
911
- ",
912
- info="Higher = more creative, Lower = more focused"
913
- )
914
  with gr.Row():
915
  k = gr.Slider(
916
- 1, 100, value=40, step=1 with gr.Row():
917
- k = gr.Slider(
918
- 1, 100, value=40,
919
- label="Top-K",
920
- info="Number of top tokens to consider"
921
- )
922
- p = gr.Slider, step=1,
923
  label="Top-K",
924
  info="Number of top tokens to consider"
925
  )
926
  p = gr.Slider(
927
- (
928
  0.1, 1.0, value=0.9, step=0.05,
929
  label="Top-P",
930
  info="Nucleus sampling threshold"
931
  )
932
- rp = gr.S0.1, 1.0, value=0.9, step=0.05,
933
- label="Top-P",
934
- info="Nucleus sampling threshold"
935
- )
936
  rp = gr.Slider(
937
- lider(
938
- 1.0,1.0, 2.0, value=1.2, step=0.1,
939
- label="Repetition Penalty",
940
- info="Penalize 2.0, value=1.2, step=0.1,
941
  label="Repetition Penalty",
942
- info repeated tokens"
943
- )
944
-
945
- # Web Search Settings (Collapsible)
946
- with gr.Accordion("🌐 Web Search Settings", open=False,="Penalize repeated tokens"
947
  )
948
 
949
  # Web Search Settings (Collapsible)
950
  with gr.Accordion("🌐 Web Search Settings", open=False, visible=False) as search_settings:
951
- mr = gr.Number(
952
- value=4, precision=0,
953
- label="Max Results",
954
- info="Number of visible=False) as search_settings:
955
  mr = gr.Number(
956
  value=4, precision=0,
957
  label="Max Results",
958
  info="Number of search results to retrieve"
959
  )
960
  mc = gr.Number(
961
- value=50, precision search results to retrieve"
962
- )
963
- mc = gr.Number(
964
- value=50, precision==0,
965
- label="Max Chars/Result",
966
- info="Character limit per search result"
967
- )
968
- st = gr.Slider(
969
- minimum=0.0, maximum=30.0, step=0.50,
970
  label="Max Chars/Result",
971
  info="Character limit per search result"
972
  )
@@ -976,25 +686,10 @@ with gr.Blocks(
976
  info="Maximum time to wait for search results"
977
  )
978
  gr.Markdown("""
979
- , value=5.0,
980
- label="Search Timeout (s)",
981
- info="Maximum time to wait for search results"
982
- )
983
- gr.Markdown("" ⚠️ **Search Engines:**
984
- - Google (primary)
985
- - DuckDuckGo (fallback)
986
- - Bing (fallback)
987
-
988
- "
989
  ⚠️ **Search Engines:**
990
  - Google (primary)
991
  - DuckDuckGo (fallback)
992
- - Bing ( SafeSearch is **OFF** for comprehensive results.
993
- """)
994
-
995
- # Actions
996
- with gr.Row():
997
- clr = gr.Button("🗑️ Clearfallback)
998
 
999
  SafeSearch is **OFF** for comprehensive results.
1000
  """)
@@ -1005,145 +700,72 @@ with gr.Blocks(
1005
 
1006
  # Right Panel - Chat Interface
1007
  with gr.Column(scale=7):
1008
- chat = Chat", variant="secondary", scale=1)
1009
-
1010
- # Right Panel - Chat Interface
1011
- with gr.Column(scale=7):
1012
- gr.Chatbot(
1013
- type="messages",
1014
- height=600,
1015
- label="💬 Conversation",
1016
- show chat = gr.Chatbot(
1017
  type="messages",
1018
  height=600,
1019
  label="💬 Conversation",
1020
  show_copy_button=True,
1021
- _copy_button=True,
1022
  avatar_images=(
1023
- "data:image avatar_images=(
1024
- "data:image/svg+xml,%3/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='40' height='40Csvg xmlns='http://www.w3.org/2000/svg' width='40' height='40'%3E%3Crect width='40' height='40' rx='20' fill='%23f093fb'/%3E%3Ctext x'%3E%3Crect width='40' height='40' rx='20' fill='%23f093fb'/%3E%3Ctext x='20' y='28' text-anchor='middle' font-size='20' fill='='20' y='28' text-anchor='middle' font-size='20' fill='white' font-family='Arial'%3E👤%3C/text%3E%3C/swhite' font-family='Arial'%3E👤%3C/text%3E%3C/svg%3E",
1025
- "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='40' height='40'vg%3E",
1026
- "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='40' height='40'%3E%3Crect width='40' height='40' rx='20' fill='%23667eea'/%3E%3Ctext x='20' y='28' text-anchor='middle' font-size='20' fill='white' font-family='Arial'%%3E%3Crect width='40' height='40' rx='20' fill='%23667eea'/%3E%3Ctext x='20' y='28' text-anchor='middle' font-size='20' fill='white' font-family='3E🤖%3C/text%3E%3C/svg%3E"
1027
- ),
1028
- Arial'%3E🤖%3C/text%3E%3C/svg%3E"
1029
  ),
1030
  bubble_full_width=False,
1031
- bubble_full_width=False,
1032
  render_markdown=True,
1033
  sanitize_html=False
1034
  )
1035
 
1036
  # Input Area
1037
  with gr.Row():
1038
- render_markdown=True,
1039
- sanitize_html=False
1040
- )
1041
-
1042
- # Input Area
1043
- with gr txt = gr.Textbox(
1044
- placeholder="💭 Type your message here... (Press Enter to.Row():
1045
  txt = gr.Textbox(
1046
  placeholder="💭 Type your message here... (Press Enter to send)",
1047
- send)",
1048
  scale=9,
1049
  container=False,
1050
  show_label=False,
1051
  lines=1,
1052
- max scale=9,
1053
- container=False,
1054
- show_label=False,
1055
- lines=1,
1056
- max_lines=5_lines=5
1057
- )
1058
- with gr.Column(scale=1, min_width=120):
1059
- submit_btn
1060
  )
1061
  with gr.Column(scale=1, min_width=120):
1062
  submit_btn = gr.Button("📤 Send", variant="primary", size="lg")
1063
- cancel_btn = gr.Button("⏹️ Stop", variant=" = gr.Button("📤 Send", variant="primary", size="lg")
1064
- cancel_btn = gr.Button("⏹️stop", visible=False, size="lg")
1065
-
1066
- # Example Prompts
1067
- gr.Examples(
1068
- Stop", variant="stop", visible=False, size="lg")
1069
 
1070
  # Example Prompts
1071
  gr.Examples(
1072
  examples=[
1073
- ["Explain examples=[
1074
  ["Explain quantum computing in simple terms"],
1075
  ["Write a Python function to calculate fibonacci numbers"],
1076
- ["What are the latest developments in AI quantum computing in simple terms"],
1077
- ["Write a Python function to calculate fibonacci numbers"],
1078
- ["? (Enable web search)"],
1079
- ["Tell me a creative story about a time traveler"],
1080
- ["Help me debug this code: def add(a,bWhat are the latest developments in AI? (Enable web search)"],
1081
  ["Tell me a creative story about a time traveler"],
1082
- ["Help me debug this code: def add(a,b):): return a+b+1"]
1083
  ],
1084
  inputs=txt,
1085
- label="💡 Example Prom return a+b+1"]
1086
- ],
1087
- inputs=txt,
1088
- label="pts"
1089
- )
1090
-
1091
- # Debug/Status Info (Collapsible)
1092
- with gr.Accordion("🔍 Debug Info", open=False):
1093
- dbg = gr💡 Example Prompts"
1094
  )
1095
 
1096
  # Debug/Status Info (Collapsible)
1097
  with gr.Accordion("🔍 Debug Info", open=False):
1098
  dbg = gr.Markdown()
1099
 
1100
- # Footer.Markdown()
1101
-
1102
  # Footer
1103
  gr.Markdown("""
1104
  ---
1105
  💡 **Tips:**
1106
- - Use **Advanced Parameters** to fine-tune creativity and response
1107
- gr.Markdown("""
1108
- ---
1109
- 💡 **Tips:**
1110
  - Use **Advanced Parameters** to fine-tune creativity and response length
1111
- length
1112
  - Enable **Web Search** for real-time information (uses multiple search engines)
1113
- - SafeSearch is **OFF** for comprehensive results - Enable **Web Search** for real-time information (uses multiple search engines)
1114
  - SafeSearch is **OFF** for comprehensive results
1115
- - Try different **
1116
- - Try different **models** for various tasks (reasonmodels** for various tasks (reasoning, coding, general chat)
1117
  - Click the **Copy** button on responses to save them to your clipboard
1118
- ing, coding, general chat)
1119
- - Click the **Copy** button on responses to save them to your clipboard
1120
- """, elem_classes="footer """, elem_classes="footer")
1121
-
1122
- # --- Event Listeners ---
1123
-
1124
- # Group all inputs")
1125
 
1126
  # --- Event Listeners ---
1127
 
1128
- # Group all for cleaner event handling
1129
  chat_inputs = [txt, chat, sys_prompt, search_chk, mr, mc, model_dd, max_tok, temp, k, p, rp, st]
1130
- # Group inputs for cleaner event handling
1131
- chat_inputs = [txt, chat, sys_prompt, search_chk, mr, mc, model_dd, max_tok, temp, k, p, rp, st all UI components that can be updated.
1132
- ui_components = [chat, dbg, txt, submit_btn, cancel_btn]
1133
  # Group all UI components that can be updated.
1134
  ui_components = [chat, dbg, txt, submit_btn, cancel_btn]
1135
 
1136
- def submit]
1137
-
1138
  def submit_and_manage_ui(user_msg, chat_history, *args):
1139
  """
1140
- Orchestrator_and_manage_ui(user_msg, chat_history, *args):
1141
- """
1142
- Orchestrator function that manages UI function that manages UI state and calls the backend chat function.
1143
- """
1144
- if not user_msg.strip():
1145
- yield {}
1146
- state and calls the backend chat function.
1147
  """
1148
  if not user_msg.strip():
1149
  yield {}
@@ -1156,175 +778,86 @@ ing, coding, general chat)
1156
  cancel_btn: gr.update(visible=True),
1157
  }
1158
 
1159
- cancelled = return
1160
-
1161
- # Update UI to "generating" state
1162
- yield {
1163
- txt: gr.update(value="", interactive=False),
1164
- submit_btn: gr.update(interactive=False),
1165
- cancel_btn: gr.update(visible=True),
1166
- }
1167
-
1168
  cancelled = False
1169
  try:
1170
  backend_args = [user_msg, chat_history] + list(args)
1171
  for response_chunk in chat_response(*backend_args):
1172
  yield {
1173
- chat False
1174
- try:
1175
- backend_args = [user_msg, chat_history] + list(args)
1176
- for response_chunk in chat_response(*backend_args):
1177
- yield: response_chunk[0],
1178
- dbg: response_chunk[1],
1179
- }
1180
- except GeneratorExit:
1181
- cancelled = True
1182
- print("Generation {
1183
  chat: response_chunk[0],
1184
  dbg: response_chunk[1],
1185
  }
1186
  except GeneratorExit:
1187
  cancelled = True
1188
- cancelled by user.")
1189
- raise
1190
- except Exception as e:
1191
- print(f"An error occurred during generation: { print("Generation cancelled by user.")
1192
  raise
1193
  except Exception as e:
1194
  print(f"An error occurred during generation: {e}")
1195
- e}")
1196
  error_history = (chat_history or []) + [
1197
  {'role': 'user', 'content': user_msg},
1198
- error_history = (chat_history or []) + [
1199
- {'role': 'user', 'content': {'role': 'assistant', 'content': f"**An error occurred:** {str user_msg},
1200
- {'role': 'assistant', 'content': f"(e)}"}
1201
- ]
1202
- yield {chat: error_history}
1203
- finally:
1204
- if not cancelled:
1205
- print("**An error occurred:** {str(e)}"}
1206
  ]
1207
  yield {chat: error_history}
1208
  finally:
1209
  if not cancelled:
1210
  print("Resetting UI state.")
1211
- yield {
1212
- txt: gr.update(interactive=True),
1213
- submit_btn: gr.update(interactive=True),
1214
- Resetting UI state.")
1215
  yield {
1216
  txt: gr.update(interactive=True),
1217
  submit_btn: gr.update(interactive=True),
1218
  cancel_btn: gr.update(visible=False),
1219
  }
1220
 
1221
- def set_cancel_flag():
1222
- """Called by the cancel button, cancel_btn: gr.update(visible=False),
1223
- }
1224
-
1225
  def set_cancel_flag():
1226
  """Called by the cancel button, sets the global event."""
1227
  cancel_event.set()
1228
  print("Cancellation signal sent.")
1229
 
1230
- def reset_ui_after sets the global event."""
1231
- cancel_event.set()
1232
- print("Cancellation signal sent.")
1233
-
1234
  def reset_ui_after_cancel():
1235
- _cancel():
1236
  """Reset UI components after cancellation."""
1237
  cancel_event.clear()
1238
  print("UI reset after cancellation.")
1239
- return {
1240
- txt: gr.update """Reset UI components after cancellation."""
1241
- cancel_event.clear()
1242
- print("UI reset after cancellation.")
1243
  return {
1244
  txt: gr.update(interactive=True),
1245
- submit_btn: gr.update(interactive=True(interactive=True),
1246
  submit_btn: gr.update(interactive=True),
1247
  cancel_btn: gr.update(visible=False),
1248
  }
1249
 
1250
- # Event for submitting text via Enter key or Submit button
1251
- submit),
1252
- cancel_btn: gr.update(visible=False),
1253
- }
1254
-
1255
  # Event for submitting text via Enter key or Submit button
1256
  submit_event = txt.submit(
1257
- fn=submit_and_event = txt.submit(
1258
  fn=submit_and_manage_ui,
1259
  inputs=chat_inputs,
1260
  outputs=ui_components,
1261
  )
1262
- _manage_ui,
1263
- inputs=chat_inputs,
1264
- outputs=ui_components,
1265
- )
1266
- submit_ submit_btn.click(
1267
- fn=submit_and_manage_ui,
1268
- inputs=chat_inputs,
1269
- outputs=ui_components,
1270
- btn.click(
1271
  fn=submit_and_manage_ui,
1272
  inputs=chat_inputs,
1273
  outputs=ui_components,
1274
  )
1275
 
1276
- # Event )
1277
-
1278
  # Event for the "Cancel" button.
1279
  cancel_btn.click(
1280
  fn=set_cancel_flag,
1281
  cancels=[submit_event]
1282
  ).then(
1283
- fn=reset for the "Cancel" button.
1284
- cancel_btn.click(
1285
- fn=set_cancel_flag,
1286
- cancels=[submit_event]
1287
- ).then(
1288
- fn=reset_ui__ui_after_cancel,
1289
- outputs=ui_components
1290
- )
1291
-
1292
- # Listeners forafter_cancel,
1293
  outputs=ui_components
1294
  )
1295
 
1296
  # Listeners for updating the duration estimate
1297
- duration updating the duration estimate
1298
  duration_inputs = [model_dd, search_chk, mr, mc, max_tok, st]
1299
- for component in duration_inputs:
1300
- component_inputs = [model_dd, search_chk, mr, mc, max_tok, st]
1301
  for component in duration_inputs:
1302
  component.change(fn=update_duration_estimate, inputs=duration_inputs, outputs=duration_display)
1303
 
1304
- # Toggle web search settings visibility
1305
- def toggle_search.change(fn=update_duration_estimate, inputs=duration_inputs, outputs=duration_display)
1306
-
1307
  # Toggle web search settings visibility
1308
  def toggle_search_settings(enabled):
1309
- _settings(enabled):
1310
  return gr.update(visible=enabled)
1311
 
1312
- search_chk.change(
1313
- fn=lambda enabled: (update_default_prompt(enabled), gr.update return gr.update(visible=enabled)
1314
-
1315
  search_chk.change(
1316
  fn=lambda enabled: (update_default_prompt(enabled), gr.update(visible=enabled)),
1317
  inputs=search_chk,
1318
  outputs=[sys_prompt, search_settings]
1319
  )
1320
 
1321
- # Clear chat(visible=enabled)),
1322
- inputs=search_chk,
1323
- outputs=[sys_prompt, search_settings]
1324
- )
1325
-
1326
- # Clear chat action action
1327
- clr.click(fn=lambda: ([], "", ""), outputs=[chat, txt
1328
  clr.click(fn=lambda: ([], "", ""), outputs=[chat, txt, dbg])
1329
 
1330
  demo.launch(share=True)
 
392
  # Handle thinking tags
393
  if not in_thought and '<think>' in text:
394
  in_thought = True
395
+ history.append({'role': 'assistant', 'content': '', 'metadata': {'title': '💭 Thought'}})
 
 
 
 
396
  assistant_message_started = True
397
+ after = text.split('<think>', 1)[1]
 
 
 
398
  thought_buf += after
399
 
400
  if '</think>' in thought_buf:
 
401
  before, after2 = thought_buf.split('</think>', 1)
402
+ history[-1]['content'] = before.strip()
 
 
403
  in_thought = False
404
  answer_buf = after2
405
+ history.append({'role': 'assistant', 'content': answer_buf})
 
 
 
406
  else:
407
+ history[-1]['content'] = thought_buf
 
 
 
 
 
408
  yield history, ""
409
  continue
410
 
411
  if in_thought:
412
  thought_buf += text
413
  if '</think>' in thought_buf:
414
+ before, after2 = thought_buf.split('</think>', 1)
 
 
 
415
  history[-1]['content'] = before.strip()
416
+ in_thought = False
 
 
 
417
  answer_buf = after2
 
 
418
  history.append({'role': 'assistant', 'content': answer_buf})
419
  else:
420
+ history[-1]['content'] = thought_buf
 
 
421
  yield history, ""
422
  continue
423
 
424
  # Stream answer
425
  if not assistant_message_started:
426
+ history.append({'role': 'assistant', 'content': ''})
 
 
 
 
 
427
  assistant_message_started = True
428
 
 
 
429
  answer_buf += text
430
  history[-1]['content'] = answer_buf.strip()
 
 
431
  yield history, ""
432
 
 
 
 
 
 
433
  # Main chat function
434
  def chat_response(
435
  user_msg: str,
436
  chat_history: List[Dict],
437
  system_prompt: str,
438
+ enable_search: bool,
 
 
 
 
439
  max_results: int,
440
  max_chars: int,
 
441
  model_name: str,
442
  max_tokens: int,
443
  temperature: float,
 
 
 
 
444
  top_k: int,
445
  top_p: float,
 
446
  repeat_penalty: float,
447
  search_timeout: float
 
 
448
  ) -> Generator[Tuple[List[Dict], str], None, None]:
 
449
  """Generate streaming chat responses with search integration"""
450
 
 
 
451
  cancel_event.clear()
452
  history = list(chat_history or [])
453
  history.append({'role': 'user', 'content': user_msg})
454
 
 
 
 
 
455
  # Perform search if enabled
456
  search_results: List[SearchResult] = []
457
  search_debug = "Web search disabled."
 
462
  search_results = SearchManager.search(
463
  user_msg,
464
  int(max_results),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
465
  int(max_chars),
466
  float(search_timeout)
467
  )
468
 
469
  if search_results:
470
+ search_debug = f"✅ Search completed - Found {len(search_results)} results\n\n" + "\n".join(
 
471
  f"- {r.format(int(max_chars))}" for r in search_results
472
  )
473
  else:
474
+ search_debug = "❌ No search results found. Check internet connection or try again."
 
 
 
475
  except Exception as e:
476
+ search_debug = f"❌ Search failed: {str(e)}"
 
 
 
 
 
 
 
477
  logger.error(f"Search error: {e}")
478
 
479
  try:
 
481
  if enable_search and search_results:
482
  enriched_prompt = PromptBuilder.build_search_context(
483
  search_results,
484
+ system_prompt,
 
 
 
485
  user_msg
486
  )
487
  else:
488
+ enriched_prompt = system_prompt.strip()
 
 
 
 
 
 
 
 
 
 
489
 
490
  # Load model
491
  pipe = ModelManager.load_pipeline(model_name)
492
 
493
  # Format prompt
494
+ prompt = PromptBuilder.format_conversation(history, enriched_prompt, pipe.tokenizer)
495
+ prompt_debug = f"\n\n--- Prompt Preview ---\n```\n{prompt[:500]}...\n```" if len(prompt) > 500 else f"\n\n--- Prompt Preview ---\n```\n{prompt}\n```"
 
496
 
497
  # Configure generation
498
  config = GenerationConfig(
499
+ max_tokens=max_tokens,
 
 
 
 
 
500
  temperature=temperature,
501
  top_k=top_k,
502
  top_p=top_p,
503
  repetition_penalty=repeat_penalty
504
  )
505
 
 
 
 
 
 
 
506
  # Setup streamer
507
  streamer = TextIteratorStreamer(
508
  pipe.tokenizer,
 
510
  skip_special_tokens=True
511
  )
512
 
513
+ # Start generation in background thread
 
 
 
 
 
 
 
514
  gen_kwargs = config.to_dict()
515
+ gen_kwargs['streamer'] = streamer
 
 
 
516
  gen_kwargs['return_full_text'] = False
517
 
 
 
518
  gen_thread = threading.Thread(
 
519
  target=pipe,
520
  args=(prompt,),
521
  kwargs=gen_kwargs
522
  )
523
  gen_thread.start()
524
 
525
+ # Yield initial state
 
 
 
 
 
 
 
 
 
 
526
  yield history, search_debug
527
 
528
  # Process stream
529
  for history_update, debug_update in StreamProcessor.process_stream(streamer, history):
 
530
  yield history_update, debug_update
531
 
532
  # Wait for completion
 
 
 
533
  gen_thread.join(timeout=5.0)
534
  yield history, search_debug + prompt_debug
535
 
 
 
 
536
  except GeneratorExit:
 
537
  logger.info("Generation cancelled by user")
538
  return
 
 
 
539
  except Exception as e:
540
  logger.error(f"Generation error: {e}")
541
+ history.append({'role': 'assistant', 'content': f"Error: {str(e)}"})
 
542
  yield history, search_debug
543
  finally:
544
  gc.collect()
545
 
546
  # Utility functions
547
  def get_model_size(model_name: str) -> float:
 
 
 
 
 
 
 
 
548
  """Get model size in billions of parameters"""
549
+ return MODELS.get(model_name, {}).get("params_b", 4.0)
 
 
 
 
 
550
 
551
  def get_duration_estimate(
552
  model_name: str,
553
  enable_search: bool,
554
  max_tokens: int,
555
  search_timeout: float
 
 
 
556
  ) -> float:
557
  """Calculate estimated GPU duration"""
558
  model_size = get_model_size(model_name)
 
 
559
  use_aot = model_size >= 2
560
 
561
+ base_duration = 20 if not use_aot else 40
 
 
 
562
  token_duration = max_tokens * 0.005
563
  search_duration = 10 if enable_search else 0
 
 
564
  aot_compilation = 20 if use_aot else 0
565
 
 
 
566
  return base_duration + token_duration + search_duration + aot_compilation
567
 
 
 
 
568
  def update_duration_estimate(
569
  model_name: str,
 
570
  enable_search: bool,
571
  max_results: int,
 
 
572
  max_chars: int,
573
  max_tokens: int,
574
  search_timeout: float
575
  ) -> str:
576
  """Format duration estimate for display"""
577
  try:
578
+ duration = get_duration_estimate(model_name, enable_search, max_tokens, search_timeout)
 
 
 
 
 
 
579
  model_size = get_model_size(model_name)
580
 
581
+ return f"""⏱ **Estimated GPU Time: {duration:.1f} seconds**
 
 
 
582
 
583
  📊 **Model Size:** {model_size:.1f}B parameters
584
+ 🔍 **Web Search:** {'Enabled (Multi-Engine)' if enable_search else 'Disabled'}"""
 
 
 
 
585
  except Exception as e:
 
 
586
  logger.error(f"Error calculating estimate: {e}")
587
  return f"⚠️ Error calculating estimate: {e}"
588
 
589
  def update_default_prompt(enable_search: bool) -> str:
590
+ """Generate default system prompt"""
 
 
 
 
591
  return "You are a helpful assistant."
592
 
593
  # ------------------------------
594
+ # Gradio UI
 
 
 
 
 
595
  # ------------------------------
596
  with gr.Blocks(
 
597
  title="LLM Inference",
 
 
 
 
 
598
  theme=gr.themes.Soft(
599
  primary_hue="blue",
600
  secondary_hue="blue",
601
  neutral_hue="slate",
602
  radius_size="lg",
 
603
  font=[gr.themes.GoogleFont("Syne"), "Arial", "sans-serif"]
604
  ),
605
  css="""
606
+ .duration-estimate { background: linear-gradient(135deg, #667eea15 0%, #764ba215 100%); border-left: 4px solid #667eea; padding: 12px; border-radius: 8px; margin: 16px 0; }
607
+ .chatbot { border-radius: 12px; box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1); }
608
+ button.primary { font-weight: 600; }
 
 
 
 
 
 
 
 
609
  .gradio-accordion { margin-bottom: 12px; }
610
  """
611
  ) as demo:
 
 
612
  # Header
613
  gr.Markdown("""
614
  # 🧠 LLM Inference with Multi-Engine Search
 
616
 
617
  with gr.Row():
618
  # Left Panel - Configuration
619
+ with gr.Column(scale=3):
 
 
 
 
 
 
 
 
 
620
  # Core Settings (Always Visible)
621
  with gr.Group():
622
+ gr.Markdown("### ⚙️ Core Settings")
623
+ model_dd = gr.Dropdown(
 
 
 
 
624
  label="🤖 Model",
625
  choices=list(MODELS.keys()),
626
  value="Qwen3-1.7B",
627
  info="Select the language model to use"
628
  )
 
 
 
629
  search_chk = gr.Checkbox(
630
  label="🔍 Enable Web Search",
631
  value=False,
632
+ info="Search across Google, DuckDuckGo, and Bing (no API required)"
 
 
 
 
 
 
 
633
  )
634
  sys_prompt = gr.Textbox(label="📝 System Prompt", lines=3, value=update_default_prompt(False), placeholder="Define the assistant's behavior and personality...")
635
 
636
+ # Duration Estimate
637
  duration_display = gr.Markdown(
638
+ value=update_duration_estimate("Qwen3-1.7B", False, 4, 50, 1024, 5.0),
 
 
 
639
  elem_classes="duration-estimate"
 
640
  )
641
 
642
  # Advanced Settings (Collapsible)
643
  with gr.Accordion("🎛️ Advanced Generation Parameters", open=False):
 
 
 
 
 
 
644
  max_tok = gr.Slider(
645
  64, 16384, value=1024, step=32,
646
  label="Max Tokens",
 
647
  info="Maximum length of generated response"
648
  )
 
 
 
649
  temp = gr.Slider(
650
  0.1, 2.0, value=0.7, step=0.1,
 
651
  label="Temperature",
652
  info="Higher = more creative, Lower = more focused"
653
  )
 
 
 
654
  with gr.Row():
655
  k = gr.Slider(
656
+ 1, 100, value=40, step=1,
 
 
 
 
 
 
657
  label="Top-K",
658
  info="Number of top tokens to consider"
659
  )
660
  p = gr.Slider(
 
661
  0.1, 1.0, value=0.9, step=0.05,
662
  label="Top-P",
663
  info="Nucleus sampling threshold"
664
  )
 
 
 
 
665
  rp = gr.Slider(
666
+ 1.0, 2.0, value=1.2, step=0.1,
 
 
 
667
  label="Repetition Penalty",
668
+ info="Penalize repeated tokens"
 
 
 
 
669
  )
670
 
671
  # Web Search Settings (Collapsible)
672
  with gr.Accordion("🌐 Web Search Settings", open=False, visible=False) as search_settings:
 
 
 
 
673
  mr = gr.Number(
674
  value=4, precision=0,
675
  label="Max Results",
676
  info="Number of search results to retrieve"
677
  )
678
  mc = gr.Number(
679
+ value=50, precision=0,
 
 
 
 
 
 
 
 
680
  label="Max Chars/Result",
681
  info="Character limit per search result"
682
  )
 
686
  info="Maximum time to wait for search results"
687
  )
688
  gr.Markdown("""
 
 
 
 
 
 
 
 
 
 
689
  ⚠️ **Search Engines:**
690
  - Google (primary)
691
  - DuckDuckGo (fallback)
692
+ - Bing (fallback)
 
 
 
 
 
693
 
694
  SafeSearch is **OFF** for comprehensive results.
695
  """)
 
700
 
701
  # Right Panel - Chat Interface
702
  with gr.Column(scale=7):
703
+ chat = gr.Chatbot(
 
 
 
 
 
 
 
 
704
  type="messages",
705
  height=600,
706
  label="💬 Conversation",
707
  show_copy_button=True,
 
708
  avatar_images=(
709
+ "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='40' height='40'%3E%3Crect width='40' height='40' rx='20' fill='%23f093fb'/%3E%3Ctext x='20' y='28' text-anchor='middle' font-size='20' fill='white' font-family='Arial'%3E👤%3C/text%3E%3C/svg%3E",
710
+ "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='40' height='40'%3E%3Crect width='40' height='40' rx='20' fill='%23667eea'/%3E%3Ctext x='20' y='28' text-anchor='middle' font-size='20' fill='white' font-family='Arial'%3E🤖%3C/text%3E%3C/svg%3E"
 
 
 
 
711
  ),
712
  bubble_full_width=False,
 
713
  render_markdown=True,
714
  sanitize_html=False
715
  )
716
 
717
  # Input Area
718
  with gr.Row():
 
 
 
 
 
 
 
719
  txt = gr.Textbox(
720
  placeholder="💭 Type your message here... (Press Enter to send)",
 
721
  scale=9,
722
  container=False,
723
  show_label=False,
724
  lines=1,
725
+ max_lines=5
 
 
 
 
 
 
 
726
  )
727
  with gr.Column(scale=1, min_width=120):
728
  submit_btn = gr.Button("📤 Send", variant="primary", size="lg")
729
+ cancel_btn = gr.Button("⏹️ Stop", variant="stop", visible=False, size="lg")
 
 
 
 
 
730
 
731
  # Example Prompts
732
  gr.Examples(
733
  examples=[
 
734
  ["Explain quantum computing in simple terms"],
735
  ["Write a Python function to calculate fibonacci numbers"],
736
+ ["What are the latest developments in AI? (Enable web search)"],
 
 
 
 
737
  ["Tell me a creative story about a time traveler"],
738
+ ["Help me debug this code: def add(a,b): return a+b+1"]
739
  ],
740
  inputs=txt,
741
+ label="💡 Example Prompts"
 
 
 
 
 
 
 
 
742
  )
743
 
744
  # Debug/Status Info (Collapsible)
745
  with gr.Accordion("🔍 Debug Info", open=False):
746
  dbg = gr.Markdown()
747
 
 
 
748
  # Footer
749
  gr.Markdown("""
750
  ---
751
  💡 **Tips:**
 
 
 
 
752
  - Use **Advanced Parameters** to fine-tune creativity and response length
 
753
  - Enable **Web Search** for real-time information (uses multiple search engines)
 
754
  - SafeSearch is **OFF** for comprehensive results
755
+ - Try different **models** for various tasks (reasoning, coding, general chat)
 
756
  - Click the **Copy** button on responses to save them to your clipboard
757
+ """, elem_classes="footer")
 
 
 
 
 
 
758
 
759
  # --- Event Listeners ---
760
 
761
+ # Group all inputs for cleaner event handling
762
  chat_inputs = [txt, chat, sys_prompt, search_chk, mr, mc, model_dd, max_tok, temp, k, p, rp, st]
 
 
 
763
  # Group all UI components that can be updated.
764
  ui_components = [chat, dbg, txt, submit_btn, cancel_btn]
765
 
 
 
766
  def submit_and_manage_ui(user_msg, chat_history, *args):
767
  """
768
+ Orchestrator function that manages UI state and calls the backend chat function.
 
 
 
 
 
 
769
  """
770
  if not user_msg.strip():
771
  yield {}
 
778
  cancel_btn: gr.update(visible=True),
779
  }
780
 
 
 
 
 
 
 
 
 
 
781
  cancelled = False
782
  try:
783
  backend_args = [user_msg, chat_history] + list(args)
784
  for response_chunk in chat_response(*backend_args):
785
  yield {
 
 
 
 
 
 
 
 
 
 
786
  chat: response_chunk[0],
787
  dbg: response_chunk[1],
788
  }
789
  except GeneratorExit:
790
  cancelled = True
791
+ print("Generation cancelled by user.")
 
 
 
792
  raise
793
  except Exception as e:
794
  print(f"An error occurred during generation: {e}")
 
795
  error_history = (chat_history or []) + [
796
  {'role': 'user', 'content': user_msg},
797
+ {'role': 'assistant', 'content': f"**An error occurred:** {str(e)}"}
 
 
 
 
 
 
 
798
  ]
799
  yield {chat: error_history}
800
  finally:
801
  if not cancelled:
802
  print("Resetting UI state.")
 
 
 
 
803
  yield {
804
  txt: gr.update(interactive=True),
805
  submit_btn: gr.update(interactive=True),
806
  cancel_btn: gr.update(visible=False),
807
  }
808
 
 
 
 
 
809
  def set_cancel_flag():
810
  """Called by the cancel button, sets the global event."""
811
  cancel_event.set()
812
  print("Cancellation signal sent.")
813
 
 
 
 
 
814
  def reset_ui_after_cancel():
 
815
  """Reset UI components after cancellation."""
816
  cancel_event.clear()
817
  print("UI reset after cancellation.")
 
 
 
 
818
  return {
819
  txt: gr.update(interactive=True),
 
820
  submit_btn: gr.update(interactive=True),
821
  cancel_btn: gr.update(visible=False),
822
  }
823
 
 
 
 
 
 
824
  # Event for submitting text via Enter key or Submit button
825
  submit_event = txt.submit(
 
826
  fn=submit_and_manage_ui,
827
  inputs=chat_inputs,
828
  outputs=ui_components,
829
  )
830
+ submit_btn.click(
 
 
 
 
 
 
 
 
831
  fn=submit_and_manage_ui,
832
  inputs=chat_inputs,
833
  outputs=ui_components,
834
  )
835
 
 
 
836
  # Event for the "Cancel" button.
837
  cancel_btn.click(
838
  fn=set_cancel_flag,
839
  cancels=[submit_event]
840
  ).then(
841
+ fn=reset_ui_after_cancel,
 
 
 
 
 
 
 
 
 
842
  outputs=ui_components
843
  )
844
 
845
  # Listeners for updating the duration estimate
 
846
  duration_inputs = [model_dd, search_chk, mr, mc, max_tok, st]
 
 
847
  for component in duration_inputs:
848
  component.change(fn=update_duration_estimate, inputs=duration_inputs, outputs=duration_display)
849
 
 
 
 
850
  # Toggle web search settings visibility
851
  def toggle_search_settings(enabled):
 
852
  return gr.update(visible=enabled)
853
 
 
 
 
854
  search_chk.change(
855
  fn=lambda enabled: (update_default_prompt(enabled), gr.update(visible=enabled)),
856
  inputs=search_chk,
857
  outputs=[sys_prompt, search_settings]
858
  )
859
 
860
+ # Clear chat action
 
 
 
 
 
 
861
  clr.click(fn=lambda: ([], "", ""), outputs=[chat, txt, dbg])
862
 
863
  demo.launch(share=True)