[ { "speaker": "host", "text": "Metarprompting is turning out to be a Metarprompting is turning out to be a" }, { "speaker": "host", "text": "very very powerful tool that everyone's very very powerful tool that everyone's very very powerful tool that everyone's" }, { "speaker": "host", "text": "It kind of actually feels using now" }, { "speaker": "guest1", "text": "It kind of actually feels using now" }, { "speaker": "guest1", "text": "It kind of actually feels" }, { "speaker": "guest1", "text": "like coding in you know 1995 like the like coding in you know 1995 like the like coding in you know 1995 like the" }, { "speaker": "guest2", "text": "tools are not all the way there" }, { "speaker": "guest2", "text": "We're tools are not all the way there" }, { "speaker": "guest2", "text": "We're tools are not all the way there" }, { "speaker": "guest3", "text": "you know in this new frontier" }, { "speaker": "guest3", "text": "But you know in this new frontier" }, { "speaker": "guest3", "text": "But you know in this new frontier" }, { "speaker": "host", "text": "personally it also kind of feels like personally it also kind of feels like personally it also kind of feels like" }, { "speaker": "host", "text": "learning how to manage a person where learning how to manage a person where learning how to manage a person where" }, { "speaker": "host", "text": "it's like how do I actually communicate it's like how do I actually communicate it's like how do I actually communicate" }, { "speaker": "guest1", "text": "uh you know the things that they need to uh you know the things that they need to uh you know the things that they need to" }, { "speaker": "guest1", "text": "know in order to make a good decision" }, { "speaker": "guest1", "text": "know in order to make a good decision" }, { "speaker": "guest2", "text": "know in order to make a good decision" }, { "speaker": "guest2", "text": "[Music] Welcome back to another episode of the Welcome back to another episode of the" }, { "speaker": "guest2", "text": "Today we're pulling back the light cone" }, { "speaker": "guest3", "text": "Today we're pulling back the light cone" }, { "speaker": "guest3", "text": "Today we're pulling back the" }, { "speaker": "guest3", "text": "curtain on what is actually happening curtain on what is actually happening curtain on what is actually happening" }, { "speaker": "host", "text": "inside the best AI startups when it inside the best AI startups when it inside the best AI startups when it" }, { "speaker": "host", "text": "comes to prompt engineering" }, { "speaker": "host", "text": "We surveyed comes to prompt engineering" }, { "speaker": "guest1", "text": "We surveyed comes to prompt engineering" }, { "speaker": "guest1", "text": "We surveyed" }, { "speaker": "guest1", "text": "more than a dozen companies and got more than a dozen companies and got more than a dozen companies and got" }, { "speaker": "guest2", "text": "their take right from the frontier of their take right from the frontier of their take right from the frontier of" }, { "speaker": "guest2", "text": "building this stuff, the practical tips" }, { "speaker": "guest2", "text": "building this stuff, the practical tips" }, { "speaker": "guest3", "text": "building this stuff, the practical tips" }, { "speaker": "guest3", "text": "Jared, why don't we start with an Jared, why don't we start with an Jared, why don't we start with an" }, { "speaker": "guest3", "text": "example from one of your best AI example from one of your best AI example from one of your best AI" }, { "speaker": "host", "text": "I managed to get an example startups" }, { "speaker": "host", "text": "I managed to get an example startups" }, { "speaker": "host", "text": "I managed to get an example" }, { "speaker": "guest1", "text": "from a company called Parahelp" }, { "speaker": "guest1", "text": "Parahelp from a company called Parahelp" }, { "speaker": "guest1", "text": "Parahelp from a company called Parahelp" }, { "speaker": "guest2", "text": "does AI customer support" }, { "speaker": "guest2", "text": "There are a does AI customer support" }, { "speaker": "guest2", "text": "There are a does AI customer support" }, { "speaker": "guest3", "text": "There are a" }, { "speaker": "guest3", "text": "bunch of companies who who are doing bunch of companies who who are doing bunch of companies who who are doing" }, { "speaker": "guest3", "text": "this, but Parhel is doing it really this, but Parhel is doing it really this, but Parhel is doing it really" }, { "speaker": "host", "text": "really well" }, { "speaker": "host", "text": "They're actually powering really well" }, { "speaker": "host", "text": "They're actually powering really well" }, { "speaker": "guest1", "text": "They're actually powering" }, { "speaker": "guest1", "text": "the customer support for Perplexity and the customer support for Perplexity and the customer support for Perplexity and" }, { "speaker": "guest1", "text": "Replet and Bolt and a bunch of other Replet and Bolt and a bunch of other Replet and Bolt and a bunch of other" }, { "speaker": "guest2", "text": "like top AI companies now" }, { "speaker": "guest2", "text": "So, if you if like top AI companies now" }, { "speaker": "guest2", "text": "So, if you if like top AI companies now" }, { "speaker": "guest3", "text": "So, if you if" }, { "speaker": "guest3", "text": "you go and you like email a customer you go and you like email a customer you go and you like email a customer" }, { "speaker": "guest3", "text": "support ticket into Perplexity, what's support ticket into Perplexity, what's support ticket into Perplexity, what's" }, { "speaker": "host", "text": "actually responding is like their AI actually responding is like their AI actually responding is like their AI" }, { "speaker": "host", "text": "The cool thing is that the agent" }, { "speaker": "host", "text": "The cool thing is that the agent" }, { "speaker": "guest1", "text": "The cool thing is that the" }, { "speaker": "guest1", "text": "Powerhel guys very graciously agreed to Powerhel guys very graciously agreed to Powerhel guys very graciously agreed to" }, { "speaker": "guest1", "text": "show us the actual prompt that is show us the actual prompt that is show us the actual prompt that is" }, { "speaker": "guest2", "text": "powering this agent um and to put it on powering this agent um and to put it on powering this agent um and to put it on" }, { "speaker": "guest2", "text": "screen on YouTube for the entire world screen on YouTube for the entire world screen on YouTube for the entire world" }, { "speaker": "guest2", "text": "Um it's like relatively hard to to see" }, { "speaker": "guest3", "text": "Um it's like relatively hard to to see" }, { "speaker": "guest3", "text": "Um it's like relatively hard to" }, { "speaker": "guest3", "text": "get these prompts for vertical AI agents get these prompts for vertical AI agents get these prompts for vertical AI agents" }, { "speaker": "host", "text": "because they're kind of like the crown because they're kind of like the crown because they're kind of like the crown" }, { "speaker": "host", "text": "jewels of the IP of these companies and jewels of the IP of these companies and jewels of the IP of these companies and" }, { "speaker": "host", "text": "so very grateful to the Powerhel guys so very grateful to the Powerhel guys so very grateful to the Powerhel guys" }, { "speaker": "guest1", "text": "for agreeing to basically like open for agreeing to basically like open for agreeing to basically like open" }, { "speaker": "guest1", "text": "source this prompt" }, { "speaker": "guest1", "text": "Diana, can you walk source this prompt" }, { "speaker": "guest2", "text": "Diana, can you walk source this prompt" }, { "speaker": "guest2", "text": "Diana, can you walk" }, { "speaker": "guest2", "text": "us through this very detailed prompt" }, { "speaker": "guest3", "text": "us through this very detailed prompt" }, { "speaker": "guest3", "text": "us through this very detailed prompt" }, { "speaker": "guest3", "text": "It's super interesting and it's very It's super interesting and it's very It's super interesting and it's very" }, { "speaker": "host", "text": "rare to get a chance to see this in rare to get a chance to see this in rare to get a chance to see this in" }, { "speaker": "host", "text": "So the interesting thing about action" }, { "speaker": "host", "text": "So the interesting thing about action" }, { "speaker": "guest1", "text": "So the interesting thing about" }, { "speaker": "guest1", "text": "this prompt is actually first it's this prompt is actually first it's this prompt is actually first it's" }, { "speaker": "guest1", "text": "really long" }, { "speaker": "guest2", "text": "It's very detailed in this really long" }, { "speaker": "guest2", "text": "It's very detailed in this really long" }, { "speaker": "guest2", "text": "It's very detailed in this" }, { "speaker": "guest3", "text": "document you can see is like six pages document you can see is like six pages document you can see is like six pages" }, { "speaker": "guest3", "text": "long just scrolling through it" }, { "speaker": "guest3", "text": "The big long just scrolling through it" }, { "speaker": "host", "text": "The big long just scrolling through it" }, { "speaker": "host", "text": "thing that a lot of the best prompts thing that a lot of the best prompts thing that a lot of the best prompts" }, { "speaker": "host", "text": "start with is this concept of uh setting start with is this concept of uh setting start with is this concept of uh setting" }, { "speaker": "guest1", "text": "up the role of the LLM" }, { "speaker": "guest1", "text": "You're a manager up the role of the LLM" }, { "speaker": "guest1", "text": "You're a manager up the role of the LLM" }, { "speaker": "guest2", "text": "You're a manager" }, { "speaker": "guest2", "text": "of a customer service agent and it of a customer service agent and it of a customer service agent and it" }, { "speaker": "guest2", "text": "breaks down into bullet points what it breaks down into bullet points what it breaks down into bullet points what it" }, { "speaker": "guest3", "text": "needs to do" }, { "speaker": "guest3", "text": "Then the big thing is needs to do" }, { "speaker": "guest3", "text": "Then the big thing is needs to do" }, { "speaker": "host", "text": "Then the big thing is" }, { "speaker": "host", "text": "telling the the task which is to approve telling the the task which is to approve telling the the task which is to approve" }, { "speaker": "host", "text": "or reject a tool call because it's or reject a tool call because it's or reject a tool call because it's" }, { "speaker": "guest1", "text": "orchestrating agent calls from all these orchestrating agent calls from all these orchestrating agent calls from all these" }, { "speaker": "guest1", "text": "And then it gives it a bit other ones" }, { "speaker": "guest1", "text": "And then it gives it a bit other ones" }, { "speaker": "guest2", "text": "And then it gives it a bit" }, { "speaker": "guest2", "text": "of the highle plan" }, { "speaker": "guest2", "text": "It breaks it down of the highle plan" }, { "speaker": "guest3", "text": "It breaks it down of the highle plan" }, { "speaker": "guest3", "text": "It breaks it down" }, { "speaker": "guest3", "text": "step by step" }, { "speaker": "host", "text": "You see steps one, two, step by step" }, { "speaker": "host", "text": "You see steps one, two, step by step" }, { "speaker": "host", "text": "You see steps one, two," }, { "speaker": "guest1", "text": "three, four, five" }, { "speaker": "guest1", "text": "And then it gives three, four, five" }, { "speaker": "guest1", "text": "And then it gives three, four, five" }, { "speaker": "guest2", "text": "And then it gives" }, { "speaker": "guest2", "text": "some of the important things to keep in some of the important things to keep in some of the important things to keep in" }, { "speaker": "guest2", "text": "mind that it should not kind of go weird mind that it should not kind of go weird mind that it should not kind of go weird" }, { "speaker": "guest3", "text": "into calling different kinds of tools" }, { "speaker": "guest3", "text": "into calling different kinds of tools" }, { "speaker": "guest3", "text": "into calling different kinds of tools" }, { "speaker": "host", "text": "It tells them how to structure the It tells them how to structure the It tells them how to structure the" }, { "speaker": "host", "text": "output because a lot of things with output because a lot of things with output because a lot of things with" }, { "speaker": "host", "text": "agents is you need them to integrate agents is you need them to integrate agents is you need them to integrate" }, { "speaker": "guest1", "text": "with other agents" }, { "speaker": "guest1", "text": "So almost like gluing with other agents" }, { "speaker": "guest1", "text": "So almost like gluing with other agents" }, { "speaker": "guest2", "text": "So almost like gluing" }, { "speaker": "guest2", "text": "the API call" }, { "speaker": "guest2", "text": "So the is important to the API call" }, { "speaker": "guest3", "text": "So the is important to the API call" }, { "speaker": "guest3", "text": "So the is important to" }, { "speaker": "guest3", "text": "specify that it's going to give certain specify that it's going to give certain specify that it's going to give certain" }, { "speaker": "host", "text": "uh output of accepting or rejecting and uh output of accepting or rejecting and uh output of accepting or rejecting and" }, { "speaker": "host", "text": "in this format" }, { "speaker": "host", "text": "Then this is sort of the in this format" }, { "speaker": "guest1", "text": "Then this is sort of the in this format" }, { "speaker": "guest1", "text": "Then this is sort of the" }, { "speaker": "guest1", "text": "highle section and one thing that the highle section and one thing that the highle section and one thing that the" }, { "speaker": "guest2", "text": "best prompts do they break it down sort best prompts do they break it down sort best prompts do they break it down sort" }, { "speaker": "guest2", "text": "of in this markdown type of style uh of in this markdown type of style uh of in this markdown type of style uh" }, { "speaker": "guest2", "text": "So you have sort of the formatting" }, { "speaker": "guest3", "text": "So you have sort of the formatting" }, { "speaker": "guest3", "text": "So you have sort of the" }, { "speaker": "guest3", "text": "heading here and then later on it goes heading here and then later on it goes heading here and then later on it goes" }, { "speaker": "host", "text": "into more details on how to do the into more details on how to do the into more details on how to do the" }, { "speaker": "host", "text": "planning and you see this is like a sub planning and you see this is like a sub planning and you see this is like a sub" }, { "speaker": "host", "text": "bullet part of it and as part of the bullet part of it and as part of the bullet part of it and as part of the" }, { "speaker": "guest1", "text": "plan there's actually three big sections plan there's actually three big sections plan there's actually three big sections" }, { "speaker": "guest1", "text": "is how to plan and then how to create is how to plan and then how to create is how to plan and then how to create" }, { "speaker": "guest1", "text": "each of the steps in the plan and then each of the steps in the plan and then each of the steps in the plan and then" }, { "speaker": "guest2", "text": "the highle example of the plan" }, { "speaker": "guest2", "text": "One big the highle example of the plan" }, { "speaker": "guest2", "text": "One big the highle example of the plan" }, { "speaker": "guest3", "text": "thing about the best prompts is they thing about the best prompts is they thing about the best prompts is they" }, { "speaker": "guest3", "text": "outline how to reason about the task and outline how to reason about the task and outline how to reason about the task and" }, { "speaker": "guest3", "text": "then a big thing is giving it giving it then a big thing is giving it giving it then a big thing is giving it giving it" }, { "speaker": "host", "text": "an example and this is what it does" }, { "speaker": "host", "text": "And an example and this is what it does" }, { "speaker": "host", "text": "And an example and this is what it does" }, { "speaker": "guest1", "text": "one thing that's interesting about this one thing that's interesting about this one thing that's interesting about this" }, { "speaker": "guest1", "text": "it it looks more like programming than it it looks more like programming than it it looks more like programming than" }, { "speaker": "guest1", "text": "writing English because it has this uh writing English because it has this uh writing English because it has this uh" }, { "speaker": "guest2", "text": "XML tag kind of format to specify sort XML tag kind of format to specify sort XML tag kind of format to specify sort" }, { "speaker": "guest2", "text": "of the plan" }, { "speaker": "guest2", "text": "We found that it makes it a of the plan" }, { "speaker": "guest3", "text": "We found that it makes it a of the plan" }, { "speaker": "guest3", "text": "We found that it makes it a" }, { "speaker": "guest3", "text": "lot easier for LMS to follow because a lot easier for LMS to follow because a lot easier for LMS to follow because a" }, { "speaker": "host", "text": "lot of LMS were post-trained in LHF with lot of LMS were post-trained in LHF with lot of LMS were post-trained in LHF with" }, { "speaker": "host", "text": "kind of XML type of input and it turns kind of XML type of input and it turns kind of XML type of input and it turns" }, { "speaker": "host", "text": "out to produce better results" }, { "speaker": "guest1", "text": "One out to produce better results" }, { "speaker": "guest1", "text": "One out to produce better results" }, { "speaker": "guest1", "text": "thing I'm surprised that isn't in here thing I'm surprised that isn't in here thing I'm surprised that isn't in here" }, { "speaker": "guest2", "text": "or maybe this is just the version that or maybe this is just the version that or maybe this is just the version that" }, { "speaker": "guest2", "text": "they released" }, { "speaker": "guest2", "text": "What I almost expect is they released" }, { "speaker": "guest3", "text": "What I almost expect is they released" }, { "speaker": "guest3", "text": "What I almost expect is" }, { "speaker": "guest3", "text": "there to be a section where it describes there to be a section where it describes there to be a section where it describes" }, { "speaker": "host", "text": "a particular scenario and uh actually a particular scenario and uh actually a particular scenario and uh actually" }, { "speaker": "host", "text": "gives example output for that scenario" }, { "speaker": "host", "text": "gives example output for that scenario" }, { "speaker": "guest1", "text": "gives example output for that scenario" }, { "speaker": "guest1", "text": "That's in like the next stage of the That's in like the next stage of the That's in like the next stage of the" }, { "speaker": "guest1", "text": "Because it's customer specific, right" }, { "speaker": "guest2", "text": "Because it's customer specific, right" }, { "speaker": "guest2", "text": "Because it's customer specific, right" }, { "speaker": "guest2", "text": "Because like every customer has their Because like every customer has their Because like every customer has their" }, { "speaker": "guest3", "text": "own like flavor of how to respond to own like flavor of how to respond to own like flavor of how to respond to" }, { "speaker": "guest3", "text": "these support tickets" }, { "speaker": "guest3", "text": "And so their these support tickets" }, { "speaker": "host", "text": "And so their these support tickets" }, { "speaker": "host", "text": "And so their" }, { "speaker": "host", "text": "challenge like a lot of these agent challenge like a lot of these agent challenge like a lot of these agent" }, { "speaker": "guest1", "text": "companies is like how do you build a companies is like how do you build a companies is like how do you build a" }, { "speaker": "guest1", "text": "general purpose product when every general purpose product when every general purpose product when every" }, { "speaker": "guest1", "text": "customer like wants you know has like customer like wants you know has like customer like wants you know has like" }, { "speaker": "guest2", "text": "slightly different workflows and like slightly different workflows and like slightly different workflows and like" }, { "speaker": "guest2", "text": "preferences" }, { "speaker": "guest2", "text": "has a really interesting preferences" }, { "speaker": "guest3", "text": "has a really interesting preferences" }, { "speaker": "guest3", "text": "has a really interesting" }, { "speaker": "guest3", "text": "thing that I see the vertical AI agent thing that I see the vertical AI agent thing that I see the vertical AI agent" }, { "speaker": "host", "text": "companies talking about a lot which is companies talking about a lot which is companies talking about a lot which is" }, { "speaker": "host", "text": "like how do you have enough flexibility like how do you have enough flexibility like how do you have enough flexibility" }, { "speaker": "host", "text": "to make special purpose logic without to make special purpose logic without to make special purpose logic without" }, { "speaker": "guest1", "text": "turning into a consulting company where turning into a consulting company where turning into a consulting company where" }, { "speaker": "guest1", "text": "you're building like a new prompt for you're building like a new prompt for you're building like a new prompt for" }, { "speaker": "guest1", "text": "for for every customer" }, { "speaker": "guest2", "text": "I actually think for for every customer" }, { "speaker": "guest2", "text": "I actually think for for every customer" }, { "speaker": "guest2", "text": "I actually think" }, { "speaker": "guest3", "text": "this like concept of like forking and this like concept of like forking and this like concept of like forking and" }, { "speaker": "guest3", "text": "merging prompts across customers and merging prompts across customers and merging prompts across customers and" }, { "speaker": "guest3", "text": "which part of the prompt is customer which part of the prompt is customer which part of the prompt is customer" }, { "speaker": "host", "text": "specific versus like companywide is like specific versus like companywide is like specific versus like companywide is like" }, { "speaker": "host", "text": "a like a really interesting thing that a like a really interesting thing that a like a really interesting thing that" }, { "speaker": "host", "text": "the world is only just beginning to the world is only just beginning to the world is only just beginning to" }, { "speaker": "guest1", "text": "Yeah, that's a very good point explore" }, { "speaker": "guest1", "text": "Yeah, that's a very good point explore" }, { "speaker": "guest1", "text": "Yeah, that's a very good point" }, { "speaker": "guest2", "text": "So this is concept of uh defining Jared" }, { "speaker": "guest2", "text": "So this is concept of uh defining Jared" }, { "speaker": "guest2", "text": "So this is concept of uh defining" }, { "speaker": "guest3", "text": "the prompt in the system prompt" }, { "speaker": "guest3", "text": "Then the prompt in the system prompt" }, { "speaker": "guest3", "text": "Then the prompt in the system prompt" }, { "speaker": "host", "text": "there's a de developer prompt and then there's a de developer prompt and then there's a de developer prompt and then" }, { "speaker": "host", "text": "there's a user prompt" }, { "speaker": "host", "text": "So what this mean there's a user prompt" }, { "speaker": "guest1", "text": "So what this mean there's a user prompt" }, { "speaker": "guest1", "text": "So what this mean" }, { "speaker": "guest1", "text": "is uh the system prompt is basically is uh the system prompt is basically is uh the system prompt is basically" }, { "speaker": "guest2", "text": "almost like defining uh sort of the almost like defining uh sort of the almost like defining uh sort of the" }, { "speaker": "guest2", "text": "highle API of how your company operates" }, { "speaker": "guest2", "text": "highle API of how your company operates" }, { "speaker": "guest3", "text": "highle API of how your company operates" }, { "speaker": "guest3", "text": "In this case the example of parhel is In this case the example of parhel is In this case the example of parhel is" }, { "speaker": "guest3", "text": "very much a system prompt" }, { "speaker": "host", "text": "There's very much a system prompt" }, { "speaker": "host", "text": "There's very much a system prompt" }, { "speaker": "host", "text": "nothing specific about the customer" }, { "speaker": "guest1", "text": "And nothing specific about the customer" }, { "speaker": "guest1", "text": "And nothing specific about the customer" }, { "speaker": "guest1", "text": "then as they add specific instances of then as they add specific instances of then as they add specific instances of" }, { "speaker": "guest2", "text": "that API and calling it then they stuff that API and calling it then they stuff that API and calling it then they stuff" }, { "speaker": "guest2", "text": "all that in into more the developer all that in into more the developer all that in into more the developer" }, { "speaker": "guest2", "text": "prompt which is not shown here and prompt which is not shown here and prompt which is not shown here and" }, { "speaker": "guest3", "text": "that's adds all the context of let's say that's adds all the context of let's say that's adds all the context of let's say" }, { "speaker": "guest3", "text": "working with perplexity there's certain working with perplexity there's certain working with perplexity there's certain" }, { "speaker": "guest3", "text": "ways of how you handle rack questions as ways of how you handle rack questions as ways of how you handle rack questions as" }, { "speaker": "host", "text": "opposed to working with bold is very opposed to working with bold is very opposed to working with bold is very" }, { "speaker": "host", "text": "different right and then I don't think different right and then I don't think different right and then I don't think" }, { "speaker": "host", "text": "parhelp has a user prompt because their parhelp has a user prompt because their parhelp has a user prompt because their" }, { "speaker": "guest1", "text": "product is not consumed directly by an product is not consumed directly by an product is not consumed directly by an" }, { "speaker": "guest1", "text": "end user but a end user prompt could be end user but a end user prompt could be end user but a end user prompt could be" }, { "speaker": "guest1", "text": "more like replet or a zero right where more like replet or a zero right where more like replet or a zero right where" }, { "speaker": "guest2", "text": "users need to type is like generate me a users need to type is like generate me a users need to type is like generate me a" }, { "speaker": "guest2", "text": "site that that has these buttons this site that that has these buttons this site that that has these buttons this" }, { "speaker": "guest2", "text": "and that that goes all in the user and that that goes all in the user and that that goes all in the user" }, { "speaker": "guest3", "text": "So that's sort of the prompt" }, { "speaker": "guest3", "text": "So that's sort of the prompt" }, { "speaker": "guest3", "text": "So that's sort of the" }, { "speaker": "host", "text": "architecture that's sort of emerging" }, { "speaker": "host", "text": "architecture that's sort of emerging" }, { "speaker": "host", "text": "architecture that's sort of emerging" }, { "speaker": "guest1", "text": "And to your point about avoiding And to your point about avoiding And to your point about avoiding" }, { "speaker": "guest1", "text": "becoming a consulting company, I think becoming a consulting company, I think becoming a consulting company, I think" }, { "speaker": "guest1", "text": "um there's so many startup opportunities um there's so many startup opportunities um there's so many startup opportunities" }, { "speaker": "guest2", "text": "and building the tooling around all of and building the tooling around all of and building the tooling around all of" }, { "speaker": "guest2", "text": "this stuff like for example like um this stuff like for example like um this stuff like for example like um" }, { "speaker": "guest2", "text": "anyone who's done prompt engineering anyone who's done prompt engineering anyone who's done prompt engineering" }, { "speaker": "guest3", "text": "knows that the examples and worked knows that the examples and worked knows that the examples and worked" }, { "speaker": "guest3", "text": "examples are really important to examples are really important to examples are really important to" }, { "speaker": "guest3", "text": "improving the quality of the output" }, { "speaker": "host", "text": "And improving the quality of the output" }, { "speaker": "host", "text": "And improving the quality of the output" }, { "speaker": "host", "text": "so then if you take like power as an so then if you take like power as an so then if you take like power as an" }, { "speaker": "guest1", "text": "example, they really want good worked example, they really want good worked example, they really want good worked" }, { "speaker": "guest1", "text": "examples that are specific to each examples that are specific to each examples that are specific to each" }, { "speaker": "guest1", "text": "And so you can imagine that as company" }, { "speaker": "guest2", "text": "And so you can imagine that as company" }, { "speaker": "guest2", "text": "And so you can imagine that as" }, { "speaker": "guest2", "text": "they scale, you almost want that done they scale, you almost want that done they scale, you almost want that done" }, { "speaker": "guest3", "text": "automatically" }, { "speaker": "guest3", "text": "Like in your dream world, automatically" }, { "speaker": "guest3", "text": "Like in your dream world, automatically" }, { "speaker": "host", "text": "Like in your dream world," }, { "speaker": "host", "text": "what you want is just like a an agent what you want is just like a an agent what you want is just like a an agent" }, { "speaker": "host", "text": "itself that can pluck out the best itself that can pluck out the best itself that can pluck out the best" }, { "speaker": "guest1", "text": "examples from like the customer data set examples from like the customer data set examples from like the customer data set" }, { "speaker": "guest1", "text": "and then software that just like ingests and then software that just like ingests and then software that just like ingests" }, { "speaker": "guest1", "text": "that straight into like wherever it that straight into like wherever it that straight into like wherever it" }, { "speaker": "guest2", "text": "should belong in the pipeline without should belong in the pipeline without should belong in the pipeline without" }, { "speaker": "guest2", "text": "you having to manually go out and plug you having to manually go out and plug you having to manually go out and plug" }, { "speaker": "guest2", "text": "that all and ingest it in all of that all and ingest it in all of that all and ingest it in all of" }, { "speaker": "guest3", "text": "That's probably a great segue yourself" }, { "speaker": "guest3", "text": "That's probably a great segue yourself" }, { "speaker": "guest3", "text": "That's probably a great segue" }, { "speaker": "host", "text": "into metaparrompting which is one of the into metaparrompting which is one of the into metaparrompting which is one of the" }, { "speaker": "host", "text": "things we want to talk about because things we want to talk about because things we want to talk about because" }, { "speaker": "host", "text": "that's that's a consistent theme that that's that's a consistent theme that that's that's a consistent theme that" }, { "speaker": "guest1", "text": "keeps coming up when we talk to our AI keeps coming up when we talk to our AI keeps coming up when we talk to our AI" }, { "speaker": "guest1", "text": "Yeah, Tropier is uh one of the startups" }, { "speaker": "guest1", "text": "Yeah, Tropier is uh one of the startups" }, { "speaker": "guest2", "text": "Yeah, Tropier is uh one of the" }, { "speaker": "guest2", "text": "startups I'm working with in the current startups I'm working with in the current startups I'm working with in the current" }, { "speaker": "guest2", "text": "YC batch and they've really helped YC batch and they've really helped YC batch and they've really helped" }, { "speaker": "guest3", "text": "people like YC company Ducky do really people like YC company Ducky do really people like YC company Ducky do really" }, { "speaker": "guest3", "text": "in-depth understanding and debugging of in-depth understanding and debugging of in-depth understanding and debugging of" }, { "speaker": "guest3", "text": "the prompts and the return values from a the prompts and the return values from a the prompts and the return values from a" }, { "speaker": "host", "text": "multi-stage workflow" }, { "speaker": "host", "text": "And one of the multi-stage workflow" }, { "speaker": "host", "text": "And one of the multi-stage workflow" }, { "speaker": "guest1", "text": "And one of the" }, { "speaker": "guest1", "text": "things they figured out is prompt things they figured out is prompt things they figured out is prompt" }, { "speaker": "guest1", "text": "So you know basically one folding" }, { "speaker": "guest2", "text": "So you know basically one folding" }, { "speaker": "guest2", "text": "So you know basically one" }, { "speaker": "guest2", "text": "prompt can dynamically generate better prompt can dynamically generate better prompt can dynamically generate better" }, { "speaker": "guest3", "text": "versions of itself" }, { "speaker": "guest3", "text": "So a good example of versions of itself" }, { "speaker": "guest3", "text": "So a good example of versions of itself" }, { "speaker": "host", "text": "So a good example of" }, { "speaker": "host", "text": "that is a classifier prompt that that is a classifier prompt that that is a classifier prompt that" }, { "speaker": "host", "text": "generates a specialized prompt based on generates a specialized prompt based on generates a specialized prompt based on" }, { "speaker": "guest1", "text": "the previous query" }, { "speaker": "guest1", "text": "And so you can the previous query" }, { "speaker": "guest1", "text": "And so you can the previous query" }, { "speaker": "guest2", "text": "And so you can" }, { "speaker": "guest2", "text": "actually go in take uh the existing actually go in take uh the existing actually go in take uh the existing" }, { "speaker": "guest2", "text": "prompt that you have and actually feed prompt that you have and actually feed prompt that you have and actually feed" }, { "speaker": "guest3", "text": "it more examples where maybe the prompt it more examples where maybe the prompt it more examples where maybe the prompt" }, { "speaker": "guest3", "text": "failed or didn't quite do what you failed or didn't quite do what you failed or didn't quite do what you" }, { "speaker": "guest3", "text": "wanted and you can actually instead of wanted and you can actually instead of wanted and you can actually instead of" }, { "speaker": "host", "text": "you having to go and rewrite the prompt, you having to go and rewrite the prompt, you having to go and rewrite the prompt," }, { "speaker": "host", "text": "you just put it into um you know the raw you just put it into um you know the raw you just put it into um you know the raw" }, { "speaker": "host", "text": "LLM and say help me make this prompt LLM and say help me make this prompt LLM and say help me make this prompt" }, { "speaker": "guest1", "text": "And because it knows itself so better" }, { "speaker": "guest1", "text": "And because it knows itself so better" }, { "speaker": "guest1", "text": "And because it knows itself so" }, { "speaker": "guest2", "text": "well, strangely um metaprompting is well, strangely um metaprompting is well, strangely um metaprompting is" }, { "speaker": "guest2", "text": "turning out to be a very very powerful turning out to be a very very powerful turning out to be a very very powerful" }, { "speaker": "guest2", "text": "tool that everyone's using now" }, { "speaker": "guest3", "text": "And the tool that everyone's using now" }, { "speaker": "guest3", "text": "And the tool that everyone's using now" }, { "speaker": "guest3", "text": "next step after uh you do sort of prompt next step after uh you do sort of prompt next step after uh you do sort of prompt" }, { "speaker": "host", "text": "folding if the task is very complex folding if the task is very complex folding if the task is very complex" }, { "speaker": "host", "text": "there's this concept of uh using there's this concept of uh using there's this concept of uh using" }, { "speaker": "host", "text": "examples and this is what Jasberry does examples and this is what Jasberry does examples and this is what Jasberry does" }, { "speaker": "guest1", "text": "is one of the companies I'm working with is one of the companies I'm working with is one of the companies I'm working with" }, { "speaker": "guest1", "text": "this batch they basically build this batch they basically build this batch they basically build" }, { "speaker": "guest1", "text": "automatic bug finding in code which is a automatic bug finding in code which is a automatic bug finding in code which is a" }, { "speaker": "guest2", "text": "lot harder and the way they do it is lot harder and the way they do it is lot harder and the way they do it is" }, { "speaker": "guest2", "text": "they feed a bunch of really hard they feed a bunch of really hard they feed a bunch of really hard" }, { "speaker": "guest2", "text": "examples that only expert programmers examples that only expert programmers examples that only expert programmers" }, { "speaker": "guest3", "text": "could do let's say if you want to find could do let's say if you want to find could do let's say if you want to find" }, { "speaker": "guest3", "text": "an N plus1 query it's actually hard for an N plus1 query it's actually hard for an N plus1 query it's actually hard for" }, { "speaker": "guest3", "text": "today for even like the best LMS to find today for even like the best LMS to find today for even like the best LMS to find" }, { "speaker": "host", "text": "those and the way to do those is they those and the way to do those is they those and the way to do those is they" }, { "speaker": "host", "text": "find parts of the code then they add find parts of the code then they add find parts of the code then they add" }, { "speaker": "host", "text": "those into the prompt a meta prompt those into the prompt a meta prompt those into the prompt a meta prompt" }, { "speaker": "guest1", "text": "that's like hey this is an example of n that's like hey this is an example of n that's like hey this is an example of n" }, { "speaker": "guest1", "text": "plus1 type of error and then that works plus1 type of error and then that works plus1 type of error and then that works" }, { "speaker": "guest1", "text": "it out and I think this pattern of it out and I think this pattern of it out and I think this pattern of" }, { "speaker": "guest2", "text": "sometimes when it's too hard to even sometimes when it's too hard to even sometimes when it's too hard to even" }, { "speaker": "guest2", "text": "kind of write a pros around it let's kind of write a pros around it let's kind of write a pros around it let's" }, { "speaker": "guest2", "text": "just give you an example that turns out just give you an example that turns out just give you an example that turns out" }, { "speaker": "guest3", "text": "to work really well because it helps LM to work really well because it helps LM to work really well because it helps LM" }, { "speaker": "guest3", "text": "to reason around complicated tasks and to reason around complicated tasks and to reason around complicated tasks and" }, { "speaker": "guest3", "text": "steer it better because you can't quite steer it better because you can't quite steer it better because you can't quite" }, { "speaker": "host", "text": "kind of put exact act parameters and kind of put exact act parameters and kind of put exact act parameters and" }, { "speaker": "host", "text": "it's almost like um unit testing it's almost like um unit testing it's almost like um unit testing" }, { "speaker": "host", "text": "programming in a sense like test-driven programming in a sense like test-driven programming in a sense like test-driven" }, { "speaker": "guest1", "text": "development is sort of the LLM v version development is sort of the LLM v version development is sort of the LLM v version" }, { "speaker": "guest1", "text": "Another thing that trope of that" }, { "speaker": "guest1", "text": "Another thing that trope of that" }, { "speaker": "guest2", "text": "Another thing that trope" }, { "speaker": "guest2", "text": "uh sort of talks about is you know the uh sort of talks about is you know the uh sort of talks about is you know the" }, { "speaker": "guest2", "text": "the model really wants to actually help the model really wants to actually help the model really wants to actually help" }, { "speaker": "guest3", "text": "you so much that if you just tell it you so much that if you just tell it you so much that if you just tell it" }, { "speaker": "guest3", "text": "give me back output in this particular give me back output in this particular give me back output in this particular" }, { "speaker": "guest3", "text": "format even if it doesn't quite have the format even if it doesn't quite have the format even if it doesn't quite have the" }, { "speaker": "host", "text": "information it needs it'll actually just information it needs it'll actually just information it needs it'll actually just" }, { "speaker": "host", "text": "tell you what it thinks you want to hear tell you what it thinks you want to hear tell you what it thinks you want to hear" }, { "speaker": "host", "text": "and it's literally a hallucination" }, { "speaker": "guest1", "text": "So, and it's literally a hallucination" }, { "speaker": "guest1", "text": "So, and it's literally a hallucination" }, { "speaker": "guest1", "text": "one thing they discovered is that you one thing they discovered is that you one thing they discovered is that you" }, { "speaker": "guest2", "text": "actually have to give the LLM's a real actually have to give the LLM's a real actually have to give the LLM's a real" }, { "speaker": "guest2", "text": "escape hatch" }, { "speaker": "guest2", "text": "You need to tell it if you escape hatch" }, { "speaker": "guest3", "text": "You need to tell it if you escape hatch" }, { "speaker": "guest3", "text": "You need to tell it if you" }, { "speaker": "guest3", "text": "do not have enough information to say do not have enough information to say do not have enough information to say" }, { "speaker": "host", "text": "yes or no or make a determination, don't yes or no or make a determination, don't yes or no or make a determination, don't" }, { "speaker": "host", "text": "just make it up" }, { "speaker": "host", "text": "Stop and ask me" }, { "speaker": "guest1", "text": "And just make it up" }, { "speaker": "guest1", "text": "Stop and ask me" }, { "speaker": "guest1", "text": "And just make it up" }, { "speaker": "guest2", "text": "Stop and ask me" }, { "speaker": "guest2", "text": "that's a very different way to think that's a very different way to think that's a very different way to think" }, { "speaker": "guest2", "text": "That's actually something we about it" }, { "speaker": "guest3", "text": "That's actually something we about it" }, { "speaker": "guest3", "text": "That's actually something we" }, { "speaker": "guest3", "text": "learned at some of the internal work learned at some of the internal work learned at some of the internal work" }, { "speaker": "host", "text": "that we've done with agents at YC where that we've done with agents at YC where that we've done with agents at YC where" }, { "speaker": "host", "text": "Jared came up with a really inventive Jared came up with a really inventive Jared came up with a really inventive" }, { "speaker": "host", "text": "way to give the LLM escape patch" }, { "speaker": "guest1", "text": "Did way to give the LLM escape patch" }, { "speaker": "guest1", "text": "Did way to give the LLM escape patch" }, { "speaker": "guest1", "text": "you want to talk about that" }, { "speaker": "guest2", "text": "So you want to talk about that" }, { "speaker": "guest2", "text": "So you want to talk about that" }, { "speaker": "guest2", "text": "the trope approach is one way to give the trope approach is one way to give the trope approach is one way to give" }, { "speaker": "guest3", "text": "the LM an escape patch" }, { "speaker": "guest3", "text": "We came up with the LM an escape patch" }, { "speaker": "guest3", "text": "We came up with the LM an escape patch" }, { "speaker": "host", "text": "We came up with" }, { "speaker": "host", "text": "a different way which is in the response a different way which is in the response a different way which is in the response" }, { "speaker": "host", "text": "format to give it the ability to have format to give it the ability to have format to give it the ability to have" }, { "speaker": "guest1", "text": "part of the response be essentially a part of the response be essentially a part of the response be essentially a" }, { "speaker": "guest1", "text": "complaint to you the developer that like complaint to you the developer that like complaint to you the developer that like" }, { "speaker": "guest1", "text": "you have given it confusing or you have given it confusing or you have given it confusing or" }, { "speaker": "guest2", "text": "underspecified information and it underspecified information and it underspecified information and it" }, { "speaker": "guest2", "text": "doesn't know what to do" }, { "speaker": "guest2", "text": "And then the doesn't know what to do" }, { "speaker": "guest3", "text": "And then the doesn't know what to do" }, { "speaker": "guest3", "text": "And then the" }, { "speaker": "guest3", "text": "nice thing about that is that we just nice thing about that is that we just nice thing about that is that we just" }, { "speaker": "host", "text": "run your LLM like in production with run your LLM like in production with run your LLM like in production with" }, { "speaker": "host", "text": "real hoser data and then you can go back real hoser data and then you can go back real hoser data and then you can go back" }, { "speaker": "host", "text": "and you can look at the outputs that it and you can look at the outputs that it and you can look at the outputs that it" }, { "speaker": "guest1", "text": "has given you in that like output has given you in that like output has given you in that like output" }, { "speaker": "guest1", "text": "Um we we call it debug info parameter" }, { "speaker": "guest1", "text": "Um we we call it debug info parameter" }, { "speaker": "guest2", "text": "Um we we call it debug info" }, { "speaker": "guest2", "text": "So like we have this like internally" }, { "speaker": "guest2", "text": "So like we have this like internally" }, { "speaker": "guest3", "text": "So like we have this like" }, { "speaker": "guest3", "text": "debug info parameter where it's debug info parameter where it's debug info parameter where it's" }, { "speaker": "guest3", "text": "basically reporting to us things that we basically reporting to us things that we basically reporting to us things that we" }, { "speaker": "host", "text": "need to fix about it and it literally need to fix about it and it literally need to fix about it and it literally" }, { "speaker": "host", "text": "ends up being like a to-do list that you ends up being like a to-do list that you ends up being like a to-do list that you" }, { "speaker": "host", "text": "the agent developer has to do" }, { "speaker": "guest1", "text": "It's like the agent developer has to do" }, { "speaker": "guest1", "text": "It's like the agent developer has to do" }, { "speaker": "guest1", "text": "really kind of mind-blowing stuff" }, { "speaker": "guest2", "text": "really kind of mind-blowing stuff" }, { "speaker": "guest2", "text": "really kind of mind-blowing stuff" }, { "speaker": "guest2", "text": "Yeah, I mean just even for hobbyists or Yeah, I mean just even for hobbyists or Yeah, I mean just even for hobbyists or" }, { "speaker": "guest3", "text": "people who are interested in playing people who are interested in playing people who are interested in playing" }, { "speaker": "guest3", "text": "around for this for personal projects" }, { "speaker": "guest3", "text": "around for this for personal projects" }, { "speaker": "host", "text": "around for this for personal projects" }, { "speaker": "host", "text": "Like a very simple way to get started Like a very simple way to get started Like a very simple way to get started" }, { "speaker": "host", "text": "with meta prompting is to follow the with meta prompting is to follow the with meta prompting is to follow the" }, { "speaker": "guest1", "text": "same structure of the prompt is give it same structure of the prompt is give it same structure of the prompt is give it" }, { "speaker": "guest1", "text": "a role and make the role be like you a role and make the role be like you a role and make the role be like you" }, { "speaker": "guest1", "text": "know you're a expert prompt engineer who know you're a expert prompt engineer who know you're a expert prompt engineer who" }, { "speaker": "guest2", "text": "gives really like detailed um great gives really like detailed um great gives really like detailed um great" }, { "speaker": "guest2", "text": "critiques and advice on how to um critiques and advice on how to um critiques and advice on how to um" }, { "speaker": "guest2", "text": "improve prompts and give it the prompt improve prompts and give it the prompt improve prompts and give it the prompt" }, { "speaker": "guest3", "text": "that you had in mind and it will spit that you had in mind and it will spit that you had in mind and it will spit" }, { "speaker": "guest3", "text": "you back a much a more expanded better you back a much a more expanded better you back a much a more expanded better" }, { "speaker": "guest3", "text": "prompt and so you can just keep running prompt and so you can just keep running prompt and so you can just keep running" }, { "speaker": "host", "text": "that loop for a while" }, { "speaker": "host", "text": "Works that loop for a while" }, { "speaker": "host", "text": "Works that loop for a while" }, { "speaker": "guest1", "text": "surprisingly well" }, { "speaker": "guest1", "text": "I think it's a common surprisingly well" }, { "speaker": "guest1", "text": "I think it's a common surprisingly well" }, { "speaker": "guest2", "text": "I think it's a common" }, { "speaker": "guest2", "text": "pattern sometimes for companies when pattern sometimes for companies when pattern sometimes for companies when" }, { "speaker": "guest2", "text": "they need to get um responses from they need to get um responses from they need to get um responses from" }, { "speaker": "guest3", "text": "element elements in their product a lot element elements in their product a lot element elements in their product a lot" }, { "speaker": "guest3", "text": "They do the meta prompting with quicker" }, { "speaker": "guest3", "text": "They do the meta prompting with quicker" }, { "speaker": "host", "text": "They do the meta prompting with" }, { "speaker": "host", "text": "a bigger beefier model any of the I a bigger beefier model any of the I a bigger beefier model any of the I" }, { "speaker": "host", "text": "don't know hundreds of billions of don't know hundreds of billions of don't know hundreds of billions of" }, { "speaker": "guest1", "text": "parameter plus models like uh I guess parameter plus models like uh I guess parameter plus models like uh I guess" }, { "speaker": "guest1", "text": "7 or your uh GPD 03 and they cloud 4 3" }, { "speaker": "guest1", "text": "7 or your uh GPD 03 and they cloud 4 3" }, { "speaker": "guest2", "text": "7 or your uh GPD 03 and they" }, { "speaker": "guest2", "text": "do this meta prompting and then they do this meta prompting and then they do this meta prompting and then they" }, { "speaker": "guest2", "text": "have a very good working one that then have a very good working one that then have a very good working one that then" }, { "speaker": "guest3", "text": "they use into the distilled model" }, { "speaker": "guest3", "text": "So they use into the distilled model" }, { "speaker": "guest3", "text": "So they use into the distilled model" }, { "speaker": "host", "text": "they use it on uh for example an FRO and they use it on uh for example an FRO and they use it on uh for example an FRO and" }, { "speaker": "host", "text": "it ends up working pretty well it ends up working pretty well it ends up working pretty well" }, { "speaker": "host", "text": "specifically sometimes for uh voice AI specifically sometimes for uh voice AI specifically sometimes for uh voice AI" }, { "speaker": "guest1", "text": "agents companies because uh latency is agents companies because uh latency is agents companies because uh latency is" }, { "speaker": "guest1", "text": "very important to uh get this whole very important to uh get this whole very important to uh get this whole" }, { "speaker": "guest1", "text": "touring test to pass because if you have touring test to pass because if you have touring test to pass because if you have" }, { "speaker": "guest2", "text": "too much pause be before the agent too much pause be before the agent too much pause be before the agent" }, { "speaker": "guest2", "text": "responds I think humans can detect responds I think humans can detect responds I think humans can detect" }, { "speaker": "guest2", "text": "something is off" }, { "speaker": "guest3", "text": "So they use a faster something is off" }, { "speaker": "guest3", "text": "So they use a faster something is off" }, { "speaker": "guest3", "text": "So they use a faster" }, { "speaker": "host", "text": "model but with a bigger better prompt model but with a bigger better prompt model but with a bigger better prompt" }, { "speaker": "host", "text": "that was refined from the bigger models" }, { "speaker": "host", "text": "that was refined from the bigger models" }, { "speaker": "guest1", "text": "that was refined from the bigger models" }, { "speaker": "guest1", "text": "So that's like a common pattern as well" }, { "speaker": "guest1", "text": "So that's like a common pattern as well" }, { "speaker": "guest2", "text": "So that's like a common pattern as well" }, { "speaker": "guest2", "text": "Another again less sophisticated maybe Another again less sophisticated maybe Another again less sophisticated maybe" }, { "speaker": "guest2", "text": "but um like as the prompt gets longer but um like as the prompt gets longer but um like as the prompt gets longer" }, { "speaker": "guest3", "text": "and longer like it becomes a a large and longer like it becomes a a large and longer like it becomes a a large" }, { "speaker": "guest3", "text": "working doc um one thing I found useful working doc um one thing I found useful working doc um one thing I found useful" }, { "speaker": "guest3", "text": "is as you're using it if you just note is as you're using it if you just note is as you're using it if you just note" }, { "speaker": "host", "text": "down in a Google doc things that you're down in a Google doc things that you're down in a Google doc things that you're" }, { "speaker": "host", "text": "seeing just um the outputs not being how seeing just um the outputs not being how seeing just um the outputs not being how" }, { "speaker": "host", "text": "you want or not ways that you can think you want or not ways that you can think you want or not ways that you can think" }, { "speaker": "guest1", "text": "of to improve it" }, { "speaker": "guest1", "text": "you can just write of to improve it" }, { "speaker": "guest1", "text": "you can just write of to improve it" }, { "speaker": "guest2", "text": "you can just write" }, { "speaker": "guest2", "text": "those in note form and then give Gemini those in note form and then give Gemini those in note form and then give Gemini" }, { "speaker": "guest2", "text": "Pro like your notes plus the original Pro like your notes plus the original Pro like your notes plus the original" }, { "speaker": "guest3", "text": "prompt and ask it to suggest a bunch of prompt and ask it to suggest a bunch of prompt and ask it to suggest a bunch of" }, { "speaker": "guest3", "text": "edits to the prompt um to incorporate edits to the prompt um to incorporate edits to the prompt um to incorporate" }, { "speaker": "guest3", "text": "these in well and it does that quite these in well and it does that quite these in well and it does that quite" }, { "speaker": "host", "text": "The other trick is uh in uh Gemini well" }, { "speaker": "host", "text": "The other trick is uh in uh Gemini well" }, { "speaker": "host", "text": "The other trick is uh in uh Gemini" }, { "speaker": "guest1", "text": "5 Pro if you look at the thinking 2" }, { "speaker": "guest1", "text": "5 Pro if you look at the thinking 2" }, { "speaker": "guest1", "text": "5 Pro if you look at the thinking" }, { "speaker": "guest2", "text": "traces as is uh parsing through uh traces as is uh parsing through uh traces as is uh parsing through uh" }, { "speaker": "guest2", "text": "evaluation you could actually learn a evaluation you could actually learn a evaluation you could actually learn a" }, { "speaker": "guest2", "text": "lot about all those misses as well" }, { "speaker": "guest3", "text": "lot about all those misses as well" }, { "speaker": "guest3", "text": "lot about all those misses as well" }, { "speaker": "guest3", "text": "We've done that internal as well, right" }, { "speaker": "host", "text": "We've done that internal as well, right" }, { "speaker": "host", "text": "We've done that internal as well, right" }, { "speaker": "host", "text": "As this is critical because if you're As this is critical because if you're As this is critical because if you're" }, { "speaker": "guest1", "text": "just using Gemini via the API until just using Gemini via the API until just using Gemini via the API until" }, { "speaker": "guest1", "text": "recently, you did not get the thinking recently, you did not get the thinking recently, you did not get the thinking" }, { "speaker": "guest1", "text": "traces and like the thinking traces are traces and like the thinking traces are traces and like the thinking traces are" }, { "speaker": "guest2", "text": "like the critical debug information to like the critical debug information to like the critical debug information to" }, { "speaker": "guest2", "text": "like understand like what's wrong with like understand like what's wrong with like understand like what's wrong with" }, { "speaker": "guest2", "text": "your prompt" }, { "speaker": "guest3", "text": "They just added it to the your prompt" }, { "speaker": "guest3", "text": "They just added it to the your prompt" }, { "speaker": "guest3", "text": "They just added it to the" }, { "speaker": "host", "text": "So you can now actually like pipe API" }, { "speaker": "host", "text": "So you can now actually like pipe API" }, { "speaker": "host", "text": "So you can now actually like pipe" }, { "speaker": "guest1", "text": "that back into your developer tools and that back into your developer tools and that back into your developer tools and" }, { "speaker": "guest1", "text": "Yeah, I think it's an workflows" }, { "speaker": "guest1", "text": "Yeah, I think it's an workflows" }, { "speaker": "guest2", "text": "Yeah, I think it's an" }, { "speaker": "guest2", "text": "underrated um consequence of Gemini Pro underrated um consequence of Gemini Pro underrated um consequence of Gemini Pro" }, { "speaker": "guest2", "text": "having such long context windows is you having such long context windows is you having such long context windows is you" }, { "speaker": "guest3", "text": "can effectively use it like a a ripple" }, { "speaker": "guest3", "text": "can effectively use it like a a ripple" }, { "speaker": "guest3", "text": "can effectively use it like a a ripple" }, { "speaker": "host", "text": "Go sort of like one by one like put your Go sort of like one by one like put your Go sort of like one by one like put your" }, { "speaker": "host", "text": "prompt on like one example then prompt on like one example then prompt on like one example then" }, { "speaker": "host", "text": "literally watch the reasoning trace in literally watch the reasoning trace in literally watch the reasoning trace in" }, { "speaker": "guest1", "text": "real time to figure out like how you can real time to figure out like how you can real time to figure out like how you can" }, { "speaker": "guest1", "text": "steer it in the direction you want" }, { "speaker": "guest1", "text": "steer it in the direction you want" }, { "speaker": "guest2", "text": "steer it in the direction you want" }, { "speaker": "guest2", "text": "Jared and the software team at YC has Jared and the software team at YC has Jared and the software team at YC has" }, { "speaker": "guest2", "text": "actually built this um you know various actually built this um you know various actually built this um you know various" }, { "speaker": "guest3", "text": "forms of workbenches that allow us to forms of workbenches that allow us to forms of workbenches that allow us to" }, { "speaker": "guest3", "text": "like do debug and things like that" }, { "speaker": "guest3", "text": "But like do debug and things like that" }, { "speaker": "host", "text": "But like do debug and things like that" }, { "speaker": "host", "text": "to your point like sometimes it's better to your point like sometimes it's better to your point like sometimes it's better" }, { "speaker": "host", "text": "just to use just to use just to use" }, { "speaker": "guest1", "text": "com directly and then drag gemini" }, { "speaker": "guest1", "text": "com directly and then drag gemini" }, { "speaker": "guest1", "text": "com directly and then drag" }, { "speaker": "guest2", "text": "and drop you know literally JSON files and drop you know literally JSON files and drop you know literally JSON files" }, { "speaker": "guest2", "text": "and uh you know you don't have to do it and uh you know you don't have to do it and uh you know you don't have to do it" }, { "speaker": "guest2", "text": "in some sort of special container like in some sort of special container like in some sort of special container like" }, { "speaker": "guest3", "text": "it you know seems to be totally it you know seems to be totally it you know seems to be totally" }, { "speaker": "guest3", "text": "something that works even directly in something that works even directly in something that works even directly in" }, { "speaker": "guest3", "text": "you know chat GPT itself" }, { "speaker": "host", "text": "Yeah, this is you know chat GPT itself" }, { "speaker": "host", "text": "Yeah, this is you know chat GPT itself" }, { "speaker": "host", "text": "Yeah, this is" }, { "speaker": "guest1", "text": "Um, I would give a shout out all stuff" }, { "speaker": "guest1", "text": "Um, I would give a shout out all stuff" }, { "speaker": "guest1", "text": "Um, I would give a shout out" }, { "speaker": "guest2", "text": "to YC's head of data, Eric Bacon, who's to YC's head of data, Eric Bacon, who's to YC's head of data, Eric Bacon, who's" }, { "speaker": "guest2", "text": "um, helped us all a lot a lot of this um, helped us all a lot a lot of this um, helped us all a lot a lot of this" }, { "speaker": "guest2", "text": "metaparrotting and using Gemini Pro 2" }, { "speaker": "guest3", "text": "5 metaparrotting and using Gemini Pro 2" }, { "speaker": "guest3", "text": "5 metaparrotting and using Gemini Pro 2" }, { "speaker": "guest3", "text": "as a effectively a ripple" }, { "speaker": "host", "text": "What about as a effectively a ripple" }, { "speaker": "host", "text": "What about as a effectively a ripple" }, { "speaker": "host", "text": "I mean, we've uh, talked about evals" }, { "speaker": "guest1", "text": "I mean, we've uh, talked about evals" }, { "speaker": "guest1", "text": "I mean, we've uh, talked about" }, { "speaker": "guest1", "text": "evals for going on a year now" }, { "speaker": "guest2", "text": "Um, what evals for going on a year now" }, { "speaker": "guest2", "text": "Um, what evals for going on a year now" }, { "speaker": "guest2", "text": "are some of the things that founders are are some of the things that founders are are some of the things that founders are" }, { "speaker": "guest3", "text": "discovering" }, { "speaker": "guest3", "text": "Even though we've been discovering" }, { "speaker": "guest3", "text": "Even though we've been discovering" }, { "speaker": "host", "text": "Even though we've been" }, { "speaker": "host", "text": "saying this for a year or more now, saying this for a year or more now, saying this for a year or more now," }, { "speaker": "host", "text": "Gary, I think it's still the case that Gary, I think it's still the case that Gary, I think it's still the case that" }, { "speaker": "guest1", "text": "like evals are the true crown jewel like like evals are the true crown jewel like like evals are the true crown jewel like" }, { "speaker": "guest1", "text": "data asset for all of these companies" }, { "speaker": "guest1", "text": "data asset for all of these companies" }, { "speaker": "guest2", "text": "data asset for all of these companies" }, { "speaker": "guest2", "text": "Like one one reason that Powerhel was Like one one reason that Powerhel was Like one one reason that Powerhel was" }, { "speaker": "guest2", "text": "willing to open source the prompt is willing to open source the prompt is willing to open source the prompt is" }, { "speaker": "guest3", "text": "they told me that they actually don't they told me that they actually don't they told me that they actually don't" }, { "speaker": "guest3", "text": "consider the prompts to be the crown consider the prompts to be the crown consider the prompts to be the crown" }, { "speaker": "guest3", "text": "jewels like the evals are the crown jewels like the evals are the crown jewels like the evals are the crown" }, { "speaker": "host", "text": "jewels because without the evals you jewels because without the evals you jewels because without the evals you" }, { "speaker": "host", "text": "don't know why the prompt was written don't know why the prompt was written don't know why the prompt was written" }, { "speaker": "host", "text": "the way that it was" }, { "speaker": "guest1", "text": "Um and it's very the way that it was" }, { "speaker": "guest1", "text": "Um and it's very the way that it was" }, { "speaker": "guest1", "text": "Um and it's very" }, { "speaker": "guest2", "text": "hard to improve it" }, { "speaker": "guest2", "text": "And I I think hard to improve it" }, { "speaker": "guest2", "text": "And I I think hard to improve it" }, { "speaker": "guest3", "text": "And I I think" }, { "speaker": "guest3", "text": "in abstraction you can think about you in abstraction you can think about you in abstraction you can think about you" }, { "speaker": "guest3", "text": "know YC funds a lot of companies know YC funds a lot of companies know YC funds a lot of companies" }, { "speaker": "host", "text": "especially in vertical AI and SAS and especially in vertical AI and SAS and especially in vertical AI and SAS and" }, { "speaker": "host", "text": "then you can't get the eval unless you then you can't get the eval unless you then you can't get the eval unless you" }, { "speaker": "host", "text": "sitting literally side by side with sitting literally side by side with sitting literally side by side with" }, { "speaker": "guest1", "text": "people who are doing X Y or Z knowledge people who are doing X Y or Z knowledge people who are doing X Y or Z knowledge" }, { "speaker": "guest1", "text": "you know, you need to sit next to work" }, { "speaker": "guest1", "text": "you know, you need to sit next to work" }, { "speaker": "guest2", "text": "you know, you need to sit next to" }, { "speaker": "guest2", "text": "the tractor sales regional manager and the tractor sales regional manager and the tractor sales regional manager and" }, { "speaker": "guest2", "text": "understand, well, you know, this person understand, well, you know, this person understand, well, you know, this person" }, { "speaker": "guest3", "text": "cares, you know, this is how they get cares, you know, this is how they get cares, you know, this is how they get" }, { "speaker": "guest3", "text": "This is what they care about" }, { "speaker": "guest3", "text": "This is what they care about" }, { "speaker": "host", "text": "This is what they care about" }, { "speaker": "host", "text": "This is that person's reward function" }, { "speaker": "host", "text": "This is that person's reward function" }, { "speaker": "guest1", "text": "This is that person's reward function" }, { "speaker": "guest1", "text": "And then you know what you're doing is And then you know what you're doing is And then you know what you're doing is" }, { "speaker": "guest1", "text": "taking these in-person interactions taking these in-person interactions taking these in-person interactions" }, { "speaker": "guest2", "text": "sitting next to someone in Nebraska and sitting next to someone in Nebraska and sitting next to someone in Nebraska and" }, { "speaker": "guest2", "text": "then going back to your computer and then going back to your computer and then going back to your computer and" }, { "speaker": "guest2", "text": "codifying it into uh very specific evals codifying it into uh very specific evals codifying it into uh very specific evals" }, { "speaker": "guest3", "text": "like this particular user wants this like this particular user wants this like this particular user wants this" }, { "speaker": "guest3", "text": "outcome after they you know after this outcome after they you know after this outcome after they you know after this" }, { "speaker": "guest3", "text": "invoice comes in we have to decide invoice comes in we have to decide invoice comes in we have to decide" }, { "speaker": "host", "text": "whether we're going to honor the you the whether we're going to honor the you the whether we're going to honor the you the" }, { "speaker": "host", "text": "warranty on this tractor" }, { "speaker": "host", "text": "Like just to warranty on this tractor" }, { "speaker": "guest1", "text": "Like just to warranty on this tractor" }, { "speaker": "guest1", "text": "Like just to" }, { "speaker": "guest1", "text": "take one of one example that's the true take one of one example that's the true take one of one example that's the true" }, { "speaker": "guest2", "text": "value right like you everyone's really value right like you everyone's really value right like you everyone's really" }, { "speaker": "guest2", "text": "worried about um are we just rappers and worried about um are we just rappers and worried about um are we just rappers and" }, { "speaker": "guest2", "text": "you know what is going to happen to you know what is going to happen to you know what is going to happen to" }, { "speaker": "guest3", "text": "startups and I think this is literally startups and I think this is literally startups and I think this is literally" }, { "speaker": "guest3", "text": "where the rubber meets the road where um where the rubber meets the road where um where the rubber meets the road where um" }, { "speaker": "guest3", "text": "if you you know if you are out there in if you you know if you are out there in if you you know if you are out there in" }, { "speaker": "host", "text": "particular places understanding that particular places understanding that particular places understanding that" }, { "speaker": "host", "text": "user better than anyone else and having user better than anyone else and having user better than anyone else and having" }, { "speaker": "host", "text": "the software actually work for those the software actually work for those the software actually work for those" }, { "speaker": "guest1", "text": "people that's the moat is that is like people that's the moat is that is like people that's the moat is that is like" }, { "speaker": "guest1", "text": "such a perfect depiction of like what is such a perfect depiction of like what is such a perfect depiction of like what is" }, { "speaker": "guest1", "text": "the core competency required of founders the core competency required of founders the core competency required of founders" }, { "speaker": "guest2", "text": "Like literally like the thing today" }, { "speaker": "guest2", "text": "Like literally like the thing today" }, { "speaker": "guest2", "text": "Like literally like the thing" }, { "speaker": "guest3", "text": "that you just said like that's your job that you just said like that's your job that you just said like that's your job" }, { "speaker": "guest3", "text": "as a founder of a company like this is as a founder of a company like this is as a founder of a company like this is" }, { "speaker": "guest3", "text": "to be really good at that thing and like to be really good at that thing and like to be really good at that thing and like" }, { "speaker": "host", "text": "maniacally obsessed with like the maniacally obsessed with like the maniacally obsessed with like the" }, { "speaker": "host", "text": "details of the regional tractor sales details of the regional tractor sales details of the regional tractor sales" }, { "speaker": "host", "text": "manager workflow" }, { "speaker": "guest1", "text": "And then the manager workflow" }, { "speaker": "guest1", "text": "And then the manager workflow" }, { "speaker": "guest1", "text": "And then the" }, { "speaker": "guest2", "text": "wild thing is it's very hard to do like wild thing is it's very hard to do like wild thing is it's very hard to do like" }, { "speaker": "guest2", "text": "you know how you have you even been to you know how you have you even been to you know how you have you even been to" }, { "speaker": "guest2", "text": "Nebraska you know the classic view is Nebraska you know the classic view is Nebraska you know the classic view is" }, { "speaker": "guest3", "text": "that uh the best founders in the world that uh the best founders in the world that uh the best founders in the world" }, { "speaker": "guest3", "text": "they're you know sort of really great they're you know sort of really great they're you know sort of really great" }, { "speaker": "guest3", "text": "cracked engineers and technologists and cracked engineers and technologists and cracked engineers and technologists and" }, { "speaker": "host", "text": "uh just really brilliant and then at the uh just really brilliant and then at the uh just really brilliant and then at the" }, { "speaker": "host", "text": "same time they have to understand some same time they have to understand some same time they have to understand some" }, { "speaker": "host", "text": "part of the world that very few people part of the world that very few people part of the world that very few people" }, { "speaker": "guest1", "text": "understand and then there's this little understand and then there's this little understand and then there's this little" }, { "speaker": "guest1", "text": "sliver that is you know uh the founder sliver that is you know uh the founder sliver that is you know uh the founder" }, { "speaker": "guest1", "text": "of a multi-billion dollar startup you of a multi-billion dollar startup you of a multi-billion dollar startup you" }, { "speaker": "guest2", "text": "know I think of Ryan Peterson from know I think of Ryan Peterson from know I think of Ryan Peterson from" }, { "speaker": "guest2", "text": "Flexport, you know, really really great Flexport, you know, really really great Flexport, you know, really really great" }, { "speaker": "guest2", "text": "person who understands how software is person who understands how software is person who understands how software is" }, { "speaker": "guest3", "text": "built, but then also I think he was the built, but then also I think he was the built, but then also I think he was the" }, { "speaker": "guest3", "text": "third biggest uh importer of medical hot third biggest uh importer of medical hot third biggest uh importer of medical hot" }, { "speaker": "guest3", "text": "tubs for an entire year like you know a tubs for an entire year like you know a tubs for an entire year like you know a" }, { "speaker": "host", "text": "So you know the weirder that decade ago" }, { "speaker": "host", "text": "So you know the weirder that decade ago" }, { "speaker": "host", "text": "So you know the weirder that" }, { "speaker": "guest1", "text": "is the more of the world that you've is the more of the world that you've is the more of the world that you've" }, { "speaker": "guest1", "text": "seen that nobody else who's a seen that nobody else who's a seen that nobody else who's a" }, { "speaker": "guest1", "text": "technologist has seen uh the greater the technologist has seen uh the greater the technologist has seen uh the greater the" }, { "speaker": "guest2", "text": "opportunity actually" }, { "speaker": "guest2", "text": "I think you've put opportunity actually" }, { "speaker": "guest2", "text": "I think you've put opportunity actually" }, { "speaker": "guest3", "text": "I think you've put" }, { "speaker": "guest3", "text": "this in a really interesting way before this in a really interesting way before this in a really interesting way before" }, { "speaker": "guest3", "text": "Gary where you're sort of saying that Gary where you're sort of saying that Gary where you're sort of saying that" }, { "speaker": "host", "text": "every founder's become a forward every founder's become a forward every founder's become a forward" }, { "speaker": "host", "text": "deployed engineer" }, { "speaker": "host", "text": "That's like a term deployed engineer" }, { "speaker": "guest1", "text": "That's like a term deployed engineer" }, { "speaker": "guest1", "text": "That's like a term" }, { "speaker": "guest1", "text": "that traces back to Palunteer and since that traces back to Palunteer and since that traces back to Palunteer and since" }, { "speaker": "guest2", "text": "you were early at Palanteer maybe tell you were early at Palanteer maybe tell you were early at Palanteer maybe tell" }, { "speaker": "guest2", "text": "us a little bit about how did forward us a little bit about how did forward us a little bit about how did forward" }, { "speaker": "guest2", "text": "deployed engineer become a thing at deployed engineer become a thing at deployed engineer become a thing at" }, { "speaker": "guest3", "text": "Palunteer and and what can founders Palunteer and and what can founders Palunteer and and what can founders" }, { "speaker": "guest3", "text": "learn from it now" }, { "speaker": "guest3", "text": "I mean I think the learn from it now" }, { "speaker": "host", "text": "I mean I think the learn from it now" }, { "speaker": "host", "text": "I mean I think the" }, { "speaker": "host", "text": "whole thesis of Palunteer at some level whole thesis of Palunteer at some level whole thesis of Palunteer at some level" }, { "speaker": "guest1", "text": "was that um if you look at Meta back was that um if you look at Meta back was that um if you look at Meta back" }, { "speaker": "guest1", "text": "then it was called Facebook or Google or then it was called Facebook or Google or then it was called Facebook or Google or" }, { "speaker": "guest1", "text": "any of the top software startups that any of the top software startups that any of the top software startups that" }, { "speaker": "guest2", "text": "everyone sort of knew back then" }, { "speaker": "guest2", "text": "One of everyone sort of knew back then" }, { "speaker": "guest2", "text": "One of everyone sort of knew back then" }, { "speaker": "guest3", "text": "the key recognitions that Peter Teal and the key recognitions that Peter Teal and the key recognitions that Peter Teal and" }, { "speaker": "guest3", "text": "Alex Karp and Stefan Cohen and Joe Alex Karp and Stefan Cohen and Joe Alex Karp and Stefan Cohen and Joe" }, { "speaker": "guest3", "text": "Lansdale, Nathan Gettings, like the Lansdale, Nathan Gettings, like the Lansdale, Nathan Gettings, like the" }, { "speaker": "host", "text": "original founders of Palunteer had was original founders of Palunteer had was original founders of Palunteer had was" }, { "speaker": "host", "text": "that uh go into anywhere in the Fortune that uh go into anywhere in the Fortune that uh go into anywhere in the Fortune" }, { "speaker": "host", "text": "500, go into any government agency in 500, go into any government agency in 500, go into any government agency in" }, { "speaker": "guest1", "text": "the world, including the United States, the world, including the United States, the world, including the United States," }, { "speaker": "guest1", "text": "and nobody who understands computer and nobody who understands computer and nobody who understands computer" }, { "speaker": "guest1", "text": "science and technology at the level that science and technology at the level that science and technology at the level that" }, { "speaker": "guest2", "text": "you at the highest possible level would you at the highest possible level would you at the highest possible level would" }, { "speaker": "guest2", "text": "ever even be in that room" }, { "speaker": "guest2", "text": "And so ever even be in that room" }, { "speaker": "guest3", "text": "And so ever even be in that room" }, { "speaker": "guest3", "text": "Palenteer's sort of really really big Palenteer's sort of really really big Palenteer's sort of really really big" }, { "speaker": "guest3", "text": "idea that they discovered very early was idea that they discovered very early was idea that they discovered very early was" }, { "speaker": "host", "text": "that uh the problems that those places that uh the problems that those places that uh the problems that those places" }, { "speaker": "host", "text": "face they're actually multi-billion face they're actually multi-billion face they're actually multi-billion" }, { "speaker": "host", "text": "dollar sometimes trillion dollar dollar sometimes trillion dollar dollar sometimes trillion dollar" }, { "speaker": "guest1", "text": "problems and yet uh this was well before problems and yet uh this was well before problems and yet uh this was well before" }, { "speaker": "guest1", "text": "AI became a thing you know I mean people AI became a thing you know I mean people AI became a thing you know I mean people" }, { "speaker": "guest1", "text": "were sort of talking about machine were sort of talking about machine were sort of talking about machine" }, { "speaker": "guest2", "text": "learning but you know back then they learning but you know back then they learning but you know back then they" }, { "speaker": "guest2", "text": "called it data mining you know the world called it data mining you know the world called it data mining you know the world" }, { "speaker": "guest2", "text": "is a wash in data these you know giant is a wash in data these you know giant is a wash in data these you know giant" }, { "speaker": "guest3", "text": "databases of people and things and databases of people and things and databases of people and things and" }, { "speaker": "guest3", "text": "transactions and we have no idea what to transactions and we have no idea what to transactions and we have no idea what to" }, { "speaker": "guest3", "text": "That's what Palanteer was, do with it" }, { "speaker": "host", "text": "That's what Palanteer was, do with it" }, { "speaker": "host", "text": "That's what Palanteer was," }, { "speaker": "host", "text": "is and still is" }, { "speaker": "guest1", "text": "That um you can go and is and still is" }, { "speaker": "guest1", "text": "That um you can go and is and still is" }, { "speaker": "guest1", "text": "That um you can go and" }, { "speaker": "guest2", "text": "find the world's best technologists who find the world's best technologists who find the world's best technologists who" }, { "speaker": "guest2", "text": "know how to write software to actually know how to write software to actually know how to write software to actually" }, { "speaker": "guest2", "text": "make sense of the world" }, { "speaker": "guest3", "text": "You know, you make sense of the world" }, { "speaker": "guest3", "text": "You know, you make sense of the world" }, { "speaker": "guest3", "text": "You know, you" }, { "speaker": "host", "text": "have these pabytes of data and you don't have these pabytes of data and you don't have these pabytes of data and you don't" }, { "speaker": "host", "text": "know how do you find the needle in the know how do you find the needle in the know how do you find the needle in the" }, { "speaker": "host", "text": "Um and you know the wild haststack" }, { "speaker": "guest1", "text": "Um and you know the wild haststack" }, { "speaker": "guest1", "text": "Um and you know the wild" }, { "speaker": "guest1", "text": "thing is going on uh something like 20 thing is going on uh something like 20 thing is going on uh something like 20" }, { "speaker": "guest2", "text": "22 years later it's only become more 22 years later it's only become more 22 years later it's only become more" }, { "speaker": "guest2", "text": "true that we have more and more data and true that we have more and more data and true that we have more and more data and" }, { "speaker": "guest2", "text": "we have less and less of an we have less and less of an we have less and less of an" }, { "speaker": "guest3", "text": "understanding of what's going on and uh understanding of what's going on and uh understanding of what's going on and uh" }, { "speaker": "guest3", "text": "it's no mistake that actually now that it's no mistake that actually now that it's no mistake that actually now that" }, { "speaker": "guest3", "text": "we have LLMs like we actually it is we have LLMs like we actually it is we have LLMs like we actually it is" }, { "speaker": "host", "text": "becoming much more tractable and then becoming much more tractable and then becoming much more tractable and then" }, { "speaker": "host", "text": "the forward deployed engineer title was the forward deployed engineer title was the forward deployed engineer title was" }, { "speaker": "host", "text": "specifically how do you sit next to specifically how do you sit next to specifically how do you sit next to" }, { "speaker": "guest1", "text": "literally the FBI agent who's um literally the FBI agent who's um literally the FBI agent who's um" }, { "speaker": "guest1", "text": "investigating domestic terrorism" }, { "speaker": "guest1", "text": "How do investigating domestic terrorism" }, { "speaker": "guest2", "text": "How do investigating domestic terrorism" }, { "speaker": "guest2", "text": "you sit right next to them in their you sit right next to them in their you sit right next to them in their" }, { "speaker": "guest2", "text": "actual office and see what does the case actual office and see what does the case actual office and see what does the case" }, { "speaker": "guest3", "text": "coming in look like" }, { "speaker": "guest3", "text": "What are all the coming in look like" }, { "speaker": "guest3", "text": "What are all the coming in look like" }, { "speaker": "host", "text": "What are all the" }, { "speaker": "host", "text": "Uh when you actually need to go steps" }, { "speaker": "host", "text": "Uh when you actually need to go steps" }, { "speaker": "guest1", "text": "Uh when you actually need to go" }, { "speaker": "guest1", "text": "to the federal prosecutor, what are the to the federal prosecutor, what are the to the federal prosecutor, what are the" }, { "speaker": "guest1", "text": "things that they're sending" }, { "speaker": "guest2", "text": "Is it I things that they're sending" }, { "speaker": "guest2", "text": "Is it I things that they're sending" }, { "speaker": "guest2", "text": "mean what's funny is like literally it's mean what's funny is like literally it's mean what's funny is like literally it's" }, { "speaker": "guest3", "text": "like word documents and Excel like word documents and Excel like word documents and Excel" }, { "speaker": "guest3", "text": "spreadsheets, right" }, { "speaker": "guest3", "text": "And um what you do spreadsheets, right" }, { "speaker": "host", "text": "And um what you do spreadsheets, right" }, { "speaker": "host", "text": "And um what you do" }, { "speaker": "host", "text": "as a forward deployed engineer is take as a forward deployed engineer is take as a forward deployed engineer is take" }, { "speaker": "guest1", "text": "these sort of you know file cabinet and these sort of you know file cabinet and these sort of you know file cabinet and" }, { "speaker": "guest1", "text": "fax machine things that people have to fax machine things that people have to fax machine things that people have to" }, { "speaker": "guest1", "text": "do and then convert it into really clean do and then convert it into really clean do and then convert it into really clean" }, { "speaker": "guest2", "text": "So you know the classic view software" }, { "speaker": "guest2", "text": "So you know the classic view software" }, { "speaker": "guest2", "text": "So you know the classic view" }, { "speaker": "guest3", "text": "is that it should be as easy to actually is that it should be as easy to actually is that it should be as easy to actually" }, { "speaker": "guest3", "text": "do uh an investigation at a threeletter do uh an investigation at a threeletter do uh an investigation at a threeletter" }, { "speaker": "guest3", "text": "agency as going and taking a photo of agency as going and taking a photo of agency as going and taking a photo of" }, { "speaker": "host", "text": "your lunch on Instagram and posting it your lunch on Instagram and posting it your lunch on Instagram and posting it" }, { "speaker": "host", "text": "to all your friends" }, { "speaker": "host", "text": "Like that's you to all your friends" }, { "speaker": "guest1", "text": "Like that's you to all your friends" }, { "speaker": "guest1", "text": "Like that's you" }, { "speaker": "guest1", "text": "know kind of the funniest part of it" }, { "speaker": "guest2", "text": "know kind of the funniest part of it" }, { "speaker": "guest2", "text": "know kind of the funniest part of it" }, { "speaker": "guest2", "text": "And so you I think it's no mistake today And so you I think it's no mistake today And so you I think it's no mistake today" }, { "speaker": "guest3", "text": "that four deployed engineers who came up that four deployed engineers who came up that four deployed engineers who came up" }, { "speaker": "guest3", "text": "through that system at Palanteer now through that system at Palanteer now through that system at Palanteer now" }, { "speaker": "guest3", "text": "they're turning out to be some of the they're turning out to be some of the they're turning out to be some of the" }, { "speaker": "host", "text": "best founders at YC actually" }, { "speaker": "host", "text": "I best founders at YC actually" }, { "speaker": "host", "text": "I best founders at YC actually" }, { "speaker": "guest1", "text": "mean produced this incredible an mean produced this incredible an mean produced this incredible an" }, { "speaker": "guest1", "text": "incredible number of startup founders incredible number of startup founders incredible number of startup founders" }, { "speaker": "guest1", "text": "cuz yeah like the training to be a fore cuz yeah like the training to be a fore cuz yeah like the training to be a fore" }, { "speaker": "guest2", "text": "deployed engineer that's exactly the deployed engineer that's exactly the deployed engineer that's exactly the" }, { "speaker": "guest2", "text": "right training to be a founder of these right training to be a founder of these right training to be a founder of these" }, { "speaker": "guest2", "text": "Now the the other interesting companies" }, { "speaker": "guest3", "text": "Now the the other interesting companies" }, { "speaker": "guest3", "text": "Now the the other interesting" }, { "speaker": "guest3", "text": "thing about Palunteer is like other thing about Palunteer is like other thing about Palunteer is like other" }, { "speaker": "host", "text": "companies would send like a salesperson companies would send like a salesperson companies would send like a salesperson" }, { "speaker": "host", "text": "to go and sit with the FBI agent and to go and sit with the FBI agent and to go and sit with the FBI agent and" }, { "speaker": "host", "text": "like Palunteer sent engineers to go and like Palunteer sent engineers to go and like Palunteer sent engineers to go and" }, { "speaker": "guest1", "text": "I think Palenter was probably do that" }, { "speaker": "guest1", "text": "I think Palenter was probably do that" }, { "speaker": "guest1", "text": "I think Palenter was probably" }, { "speaker": "guest2", "text": "the first company to really like the first company to really like the first company to really like" }, { "speaker": "guest2", "text": "institutionalize that and scale that as institutionalize that and scale that as institutionalize that and scale that as" }, { "speaker": "guest2", "text": "a process, right" }, { "speaker": "guest3", "text": "I mean, I think a process, right" }, { "speaker": "guest3", "text": "I mean, I think a process, right" }, { "speaker": "guest3", "text": "I mean, I think" }, { "speaker": "host", "text": "what happened there, the reason why they what happened there, the reason why they what happened there, the reason why they" }, { "speaker": "host", "text": "were able to get these sort of seven and were able to get these sort of seven and were able to get these sort of seven and" }, { "speaker": "host", "text": "eight and now nine figure contracts very eight and now nine figure contracts very eight and now nine figure contracts very" }, { "speaker": "guest1", "text": "consistently is that uh instead of consistently is that uh instead of consistently is that uh instead of" }, { "speaker": "guest1", "text": "sending someone who's like hair and sending someone who's like hair and sending someone who's like hair and" }, { "speaker": "guest1", "text": "teeth and they're in there and you know, teeth and they're in there and you know, teeth and they're in there and you know," }, { "speaker": "guest2", "text": "let's go to the let's go to the uh let's go to the let's go to the uh let's go to the let's go to the uh" }, { "speaker": "guest2", "text": "You know, it's all like steakhouse" }, { "speaker": "guest2", "text": "You know, it's all like steakhouse" }, { "speaker": "guest3", "text": "You know, it's all like" }, { "speaker": "guest3", "text": "relationship" }, { "speaker": "guest3", "text": "and you'd have one meeting relationship" }, { "speaker": "host", "text": "and you'd have one meeting relationship" }, { "speaker": "host", "text": "and you'd have one meeting" }, { "speaker": "host", "text": "uh they would really like the uh they would really like the uh they would really like the" }, { "speaker": "guest1", "text": "salesperson and then through sheer force salesperson and then through sheer force salesperson and then through sheer force" }, { "speaker": "guest1", "text": "of personality you'd try to get them to of personality you'd try to get them to of personality you'd try to get them to" }, { "speaker": "guest1", "text": "give you a seven-figure contract and give you a seven-figure contract and give you a seven-figure contract and" }, { "speaker": "guest2", "text": "like the time scales on this would be like the time scales on this would be like the time scales on this would be" }, { "speaker": "guest2", "text": "you know 6 weeks 10 weeks 12 weeks like you know 6 weeks 10 weeks 12 weeks like you know 6 weeks 10 weeks 12 weeks like" }, { "speaker": "guest2", "text": "5 years I don't know it's like and the 5 years I don't know it's like and the 5 years I don't know it's like and the" }, { "speaker": "guest3", "text": "software would never work uh whereas if software would never work uh whereas if software would never work uh whereas if" }, { "speaker": "guest3", "text": "you put an engineer in there and you you put an engineer in there and you you put an engineer in there and you" }, { "speaker": "guest3", "text": "give them uh you know Palunteer Foundry give them uh you know Palunteer Foundry give them uh you know Palunteer Foundry" }, { "speaker": "host", "text": "which is what they now call sort of which is what they now call sort of which is what they now call sort of" }, { "speaker": "host", "text": "their core uh data viz and data mining their core uh data viz and data mining their core uh data viz and data mining" }, { "speaker": "host", "text": "suites instead of the next meeting being suites instead of the next meeting being suites instead of the next meeting being" }, { "speaker": "guest1", "text": "reviewing 50 pages of you know sort of reviewing 50 pages of you know sort of reviewing 50 pages of you know sort of" }, { "speaker": "guest1", "text": "sales documentation or a contract or a sales documentation or a contract or a sales documentation or a contract or a" }, { "speaker": "guest1", "text": "spec or anything like that" }, { "speaker": "guest2", "text": "It's spec or anything like that" }, { "speaker": "guest2", "text": "It's spec or anything like that" }, { "speaker": "guest2", "text": "literally like, \"Okay, we built it" }, { "speaker": "guest3", "text": "\" And literally like, \"Okay, we built it" }, { "speaker": "guest3", "text": "\" And literally like, \"Okay, we built it" }, { "speaker": "guest3", "text": "then you're getting like real live then you're getting like real live then you're getting like real live" }, { "speaker": "host", "text": "feedback within days" }, { "speaker": "host", "text": "And I mean, that's feedback within days" }, { "speaker": "host", "text": "And I mean, that's feedback within days" }, { "speaker": "guest1", "text": "And I mean, that's" }, { "speaker": "guest1", "text": "honestly the biggest opportunity for honestly the biggest opportunity for honestly the biggest opportunity for" }, { "speaker": "guest1", "text": "startup founders" }, { "speaker": "guest2", "text": "If startup founders startup founders" }, { "speaker": "guest2", "text": "If startup founders startup founders" }, { "speaker": "guest2", "text": "If startup founders" }, { "speaker": "guest3", "text": "can do that and uh that's what forward can do that and uh that's what forward can do that and uh that's what forward" }, { "speaker": "guest3", "text": "deployed engineers are sort of used to deployed engineers are sort of used to deployed engineers are sort of used to" }, { "speaker": "guest3", "text": "doing that's how you could beat a doing that's how you could beat a doing that's how you could beat a" }, { "speaker": "host", "text": "Salesforce or an Oracle or you know a Salesforce or an Oracle or you know a Salesforce or an Oracle or you know a" }, { "speaker": "host", "text": "Booze Allen or literally any company out Booze Allen or literally any company out Booze Allen or literally any company out" }, { "speaker": "host", "text": "there that has a big office and a big there that has a big office and a big there that has a big office and a big" }, { "speaker": "guest1", "text": "fancy you know you have big fancy fancy you know you have big fancy fancy you know you have big fancy" }, { "speaker": "guest1", "text": "salespeople with big strong handshakes salespeople with big strong handshakes salespeople with big strong handshakes" }, { "speaker": "guest1", "text": "and it's like how does a really good and it's like how does a really good and it's like how does a really good" }, { "speaker": "guest2", "text": "engineer with a weak handshake go in engineer with a weak handshake go in engineer with a weak handshake go in" }, { "speaker": "guest2", "text": "there and beat them" }, { "speaker": "guest2", "text": "It's actually you there and beat them" }, { "speaker": "guest3", "text": "It's actually you there and beat them" }, { "speaker": "guest3", "text": "It's actually you" }, { "speaker": "guest3", "text": "show them something that they've never show them something that they've never show them something that they've never" }, { "speaker": "host", "text": "seen before and like make them feel seen before and like make them feel seen before and like make them feel" }, { "speaker": "host", "text": "super heard" }, { "speaker": "host", "text": "You have to be super super heard" }, { "speaker": "guest1", "text": "You have to be super super heard" }, { "speaker": "guest1", "text": "You have to be super" }, { "speaker": "guest1", "text": "empathetic about it" }, { "speaker": "guest2", "text": "Like you actually empathetic about it" }, { "speaker": "guest2", "text": "Like you actually empathetic about it" }, { "speaker": "guest2", "text": "Like you actually" }, { "speaker": "guest3", "text": "have to be a great designer and product have to be a great designer and product have to be a great designer and product" }, { "speaker": "guest3", "text": "person and then you know come back and person and then you know come back and person and then you know come back and" }, { "speaker": "guest3", "text": "you can just blow them away" }, { "speaker": "host", "text": "Like the you can just blow them away" }, { "speaker": "host", "text": "Like the you can just blow them away" }, { "speaker": "host", "text": "software is so powerful that you know software is so powerful that you know software is so powerful that you know" }, { "speaker": "guest1", "text": "the second you see something that you the second you see something that you the second you see something that you" }, { "speaker": "guest1", "text": "know makes you feel seen you want to buy know makes you feel seen you want to buy know makes you feel seen you want to buy" }, { "speaker": "guest1", "text": "it on the spot" }, { "speaker": "guest2", "text": "Is a good way of it on the spot" }, { "speaker": "guest2", "text": "Is a good way of it on the spot" }, { "speaker": "guest2", "text": "Is a good way of" }, { "speaker": "guest3", "text": "thinking about it that founders should thinking about it that founders should thinking about it that founders should" }, { "speaker": "guest3", "text": "think about themselves as being the four think about themselves as being the four think about themselves as being the four" }, { "speaker": "guest3", "text": "deployed engineers of their own company" }, { "speaker": "host", "text": "deployed engineers of their own company" }, { "speaker": "host", "text": "deployed engineers of their own company" }, { "speaker": "host", "text": "Like you definitely Absolutely" }, { "speaker": "guest1", "text": "Like you definitely Absolutely" }, { "speaker": "guest1", "text": "Like you definitely" }, { "speaker": "guest1", "text": "can't farm this out" }, { "speaker": "guest2", "text": "Like literally the can't farm this out" }, { "speaker": "guest2", "text": "Like literally the can't farm this out" }, { "speaker": "guest2", "text": "Like literally the" }, { "speaker": "guest3", "text": "founders themselves, they're technical" }, { "speaker": "guest3", "text": "founders themselves, they're technical" }, { "speaker": "guest3", "text": "founders themselves, they're technical" }, { "speaker": "host", "text": "They have to be the great product They have to be the great product They have to be the great product" }, { "speaker": "host", "text": "They have to be the people" }, { "speaker": "host", "text": "They have to be the people" }, { "speaker": "guest1", "text": "They have to be the" }, { "speaker": "guest1", "text": "ethnographer" }, { "speaker": "guest1", "text": "They have to be the ethnographer" }, { "speaker": "guest2", "text": "They have to be the ethnographer" }, { "speaker": "guest2", "text": "They have to be the" }, { "speaker": "guest2", "text": "You want the person on the designer" }, { "speaker": "guest3", "text": "You want the person on the designer" }, { "speaker": "guest3", "text": "You want the person on the" }, { "speaker": "guest3", "text": "second meeting to see the demo you put second meeting to see the demo you put second meeting to see the demo you put" }, { "speaker": "host", "text": "together based on the stuff you heard" }, { "speaker": "host", "text": "together based on the stuff you heard" }, { "speaker": "host", "text": "together based on the stuff you heard" }, { "speaker": "guest1", "text": "And you want them to say, \"Wow, I've And you want them to say, \"Wow, I've And you want them to say, \"Wow, I've" }, { "speaker": "guest1", "text": "never seen anything like that" }, { "speaker": "guest1", "text": "\" And take never seen anything like that" }, { "speaker": "guest2", "text": "\" And take never seen anything like that" }, { "speaker": "guest2", "text": "I think the incredible thing my money" }, { "speaker": "guest2", "text": "I think the incredible thing my money" }, { "speaker": "guest3", "text": "I think the incredible thing" }, { "speaker": "guest3", "text": "about this model is this is why we're about this model is this is why we're about this model is this is why we're" }, { "speaker": "guest3", "text": "seeing a lot of the vertical AI agents seeing a lot of the vertical AI agents seeing a lot of the vertical AI agents" }, { "speaker": "host", "text": "take off is precisely this because they take off is precisely this because they take off is precisely this because they" }, { "speaker": "host", "text": "can have these meetings with the end can have these meetings with the end can have these meetings with the end" }, { "speaker": "host", "text": "buyer and champion at these big buyer and champion at these big buyer and champion at these big" }, { "speaker": "guest1", "text": "enterprises" }, { "speaker": "guest1", "text": "They take that context and enterprises" }, { "speaker": "guest1", "text": "They take that context and enterprises" }, { "speaker": "guest2", "text": "They take that context and" }, { "speaker": "guest2", "text": "then they stuff it basically in the then they stuff it basically in the then they stuff it basically in the" }, { "speaker": "guest2", "text": "prompt and then they can quickly come prompt and then they can quickly come prompt and then they can quickly come" }, { "speaker": "guest3", "text": "back in a meeting like just the next day back in a meeting like just the next day back in a meeting like just the next day" }, { "speaker": "guest3", "text": "maybe with Palunteer would have taken a maybe with Palunteer would have taken a maybe with Palunteer would have taken a" }, { "speaker": "guest3", "text": "bit longer and a team of engineers here" }, { "speaker": "host", "text": "bit longer and a team of engineers here" }, { "speaker": "host", "text": "bit longer and a team of engineers here" }, { "speaker": "host", "text": "It could be just the two founders go in It could be just the two founders go in It could be just the two founders go in" }, { "speaker": "guest1", "text": "and then they would close this six, and then they would close this six, and then they would close this six," }, { "speaker": "guest1", "text": "seven figure deals which we've seen and seven figure deals which we've seen and seven figure deals which we've seen and" }, { "speaker": "guest1", "text": "with large enterprises which has never with large enterprises which has never with large enterprises which has never" }, { "speaker": "guest2", "text": "been done before and it's just possible been done before and it's just possible been done before and it's just possible" }, { "speaker": "guest2", "text": "with this new model of forward deploy with this new model of forward deploy with this new model of forward deploy" }, { "speaker": "guest2", "text": "engineer plus AI is just on engineer plus AI is just on engineer plus AI is just on" }, { "speaker": "guest3", "text": "accelerating" }, { "speaker": "guest3", "text": "It just reminds me of a accelerating" }, { "speaker": "guest3", "text": "It just reminds me of a accelerating" }, { "speaker": "host", "text": "It just reminds me of a" }, { "speaker": "host", "text": "company I mentioned before on the company I mentioned before on the company I mentioned before on the" }, { "speaker": "host", "text": "podcast like Giger ML who do customer podcast like Giger ML who do customer podcast like Giger ML who do customer" }, { "speaker": "guest1", "text": "another customer support and especially another customer support and especially another customer support and especially" }, { "speaker": "guest1", "text": "a lot of voice support and it's just a lot of voice support and it's just a lot of voice support and it's just" }, { "speaker": "guest1", "text": "classic case of two extremely um classic case of two extremely um classic case of two extremely um" }, { "speaker": "guest2", "text": "talented software engineers not natural talented software engineers not natural talented software engineers not natural" }, { "speaker": "guest2", "text": "sales people but they force themselves sales people but they force themselves sales people but they force themselves" }, { "speaker": "guest2", "text": "to be essentially forward deployed to be essentially forward deployed to be essentially forward deployed" }, { "speaker": "guest3", "text": "engineers and they closed a huge deal engineers and they closed a huge deal engineers and they closed a huge deal" }, { "speaker": "guest3", "text": "with Zeppto and then a couple of other with Zeppto and then a couple of other with Zeppto and then a couple of other" }, { "speaker": "guest3", "text": "companies they can't announce yet but do companies they can't announce yet but do companies they can't announce yet but do" }, { "speaker": "host", "text": "they physically go on site like the they physically go on site like the they physically go on site like the" }, { "speaker": "host", "text": "palentier model" }, { "speaker": "host", "text": "So they did so palentier model" }, { "speaker": "guest1", "text": "So they did so palentier model" }, { "speaker": "guest1", "text": "So they did so" }, { "speaker": "guest1", "text": "they they did all of that where once they they did all of that where once they they did all of that where once" }, { "speaker": "guest2", "text": "they close the deal they go on site and they close the deal they go on site and they close the deal they go on site and" }, { "speaker": "guest2", "text": "they sit there with all the customer they sit there with all the customer they sit there with all the customer" }, { "speaker": "guest2", "text": "support people and figuring out how to support people and figuring out how to support people and figuring out how to" }, { "speaker": "guest3", "text": "keep tuning and getting the software or keep tuning and getting the software or keep tuning and getting the software or" }, { "speaker": "guest3", "text": "the LM to work even better" }, { "speaker": "guest3", "text": "But before the LM to work even better" }, { "speaker": "host", "text": "But before the LM to work even better" }, { "speaker": "host", "text": "that even to win the deal what they that even to win the deal what they that even to win the deal what they" }, { "speaker": "host", "text": "found is that they can they can win by found is that they can they can win by found is that they can they can win by" }, { "speaker": "guest1", "text": "just having the most impressive demo" }, { "speaker": "guest1", "text": "just having the most impressive demo" }, { "speaker": "guest1", "text": "just having the most impressive demo" }, { "speaker": "guest2", "text": "And in their case they've um innovated a And in their case they've um innovated a And in their case they've um innovated a" }, { "speaker": "guest2", "text": "bit on the rag pipeline so that they can bit on the rag pipeline so that they can bit on the rag pipeline so that they can" }, { "speaker": "guest2", "text": "um have their voice responses be both um have their voice responses be both um have their voice responses be both" }, { "speaker": "guest3", "text": "accurate and very low latency" }, { "speaker": "guest3", "text": "sort of accurate and very low latency" }, { "speaker": "guest3", "text": "sort of accurate and very low latency" }, { "speaker": "host", "text": "like a technically challenging thing to like a technically challenging thing to like a technically challenging thing to" }, { "speaker": "host", "text": "do, but I just feel like in the like pre do, but I just feel like in the like pre do, but I just feel like in the like pre" }, { "speaker": "host", "text": "sort of the current LLM rise, you sort of the current LLM rise, you sort of the current LLM rise, you" }, { "speaker": "guest1", "text": "couldn't necessarily differentiate couldn't necessarily differentiate couldn't necessarily differentiate" }, { "speaker": "guest1", "text": "enough in the demo phase of sales to enough in the demo phase of sales to enough in the demo phase of sales to" }, { "speaker": "guest1", "text": "beat out incumbent" }, { "speaker": "guest2", "text": "So, you can really beat out incumbent" }, { "speaker": "guest2", "text": "So, you can really beat out incumbent" }, { "speaker": "guest2", "text": "So, you can really" }, { "speaker": "guest3", "text": "beat Salesforce by having a slightly beat Salesforce by having a slightly beat Salesforce by having a slightly" }, { "speaker": "guest3", "text": "better CRM with a better UI" }, { "speaker": "guest3", "text": "But now better CRM with a better UI" }, { "speaker": "host", "text": "But now better CRM with a better UI" }, { "speaker": "host", "text": "because the technology evolves so fast because the technology evolves so fast because the technology evolves so fast" }, { "speaker": "host", "text": "and it's so hard to get this like last and it's so hard to get this like last and it's so hard to get this like last" }, { "speaker": "guest1", "text": "five 10 five to 10% correct, you can five 10 five to 10% correct, you can five 10 five to 10% correct, you can" }, { "speaker": "guest1", "text": "actually if you're a forward deployed actually if you're a forward deployed actually if you're a forward deployed" }, { "speaker": "guest1", "text": "engineer go in do the first meeting engineer go in do the first meeting engineer go in do the first meeting" }, { "speaker": "guest2", "text": "tweak it so that it works really well tweak it so that it works really well tweak it so that it works really well" }, { "speaker": "guest2", "text": "for that customer" }, { "speaker": "guest2", "text": "Go back with the demo for that customer" }, { "speaker": "guest3", "text": "Go back with the demo for that customer" }, { "speaker": "guest3", "text": "Go back with the demo" }, { "speaker": "guest3", "text": "and just get that oh wow like we've not and just get that oh wow like we've not and just get that oh wow like we've not" }, { "speaker": "host", "text": "seen anyone else pull this off before seen anyone else pull this off before seen anyone else pull this off before" }, { "speaker": "host", "text": "experience and close huge deals" }, { "speaker": "host", "text": "And experience and close huge deals" }, { "speaker": "guest1", "text": "And experience and close huge deals" }, { "speaker": "guest1", "text": "that was the exact same case with Happy that was the exact same case with Happy that was the exact same case with Happy" }, { "speaker": "guest1", "text": "Robot who has sold seven figure Robot who has sold seven figure Robot who has sold seven figure" }, { "speaker": "guest2", "text": "contracts to the top three largest contracts to the top three largest contracts to the top three largest" }, { "speaker": "guest2", "text": "logistic brokers in the world" }, { "speaker": "guest2", "text": "They logistic brokers in the world" }, { "speaker": "guest3", "text": "They logistic brokers in the world" }, { "speaker": "guest3", "text": "build AI voice agents for that" }, { "speaker": "guest3", "text": "They are build AI voice agents for that" }, { "speaker": "host", "text": "They are build AI voice agents for that" }, { "speaker": "host", "text": "the ones doing the forward deploy the ones doing the forward deploy the ones doing the forward deploy" }, { "speaker": "host", "text": "engineer model and talking to like the engineer model and talking to like the engineer model and talking to like the" }, { "speaker": "guest1", "text": "CIOS of these companies and quickly CIOS of these companies and quickly CIOS of these companies and quickly" }, { "speaker": "guest1", "text": "shipping a lot of product like very very shipping a lot of product like very very shipping a lot of product like very very" }, { "speaker": "guest1", "text": "quick turnaround" }, { "speaker": "guest2", "text": "And it's been quick turnaround" }, { "speaker": "guest2", "text": "And it's been quick turnaround" }, { "speaker": "guest2", "text": "And it's been" }, { "speaker": "guest3", "text": "incredible to see that take off right incredible to see that take off right incredible to see that take off right" }, { "speaker": "guest3", "text": "And it started from six figure now" }, { "speaker": "guest3", "text": "And it started from six figure now" }, { "speaker": "host", "text": "And it started from six figure" }, { "speaker": "host", "text": "deals now doing closing and seven figure deals now doing closing and seven figure deals now doing closing and seven figure" }, { "speaker": "host", "text": "deals which is crazy" }, { "speaker": "guest1", "text": "This is just a deals which is crazy" }, { "speaker": "guest1", "text": "This is just a deals which is crazy" }, { "speaker": "guest1", "text": "This is just a" }, { "speaker": "guest2", "text": "couple months after" }, { "speaker": "guest2", "text": "So that's the kind couple months after" }, { "speaker": "guest2", "text": "So that's the kind couple months after" }, { "speaker": "guest3", "text": "So that's the kind" }, { "speaker": "guest3", "text": "of stuff that you can do with uh I mean of stuff that you can do with uh I mean of stuff that you can do with uh I mean" }, { "speaker": "guest3", "text": "unbelievably very very smart prompt unbelievably very very smart prompt unbelievably very very smart prompt" }, { "speaker": "host", "text": "engineering actually" }, { "speaker": "host", "text": "Well, one of the engineering actually" }, { "speaker": "host", "text": "Well, one of the engineering actually" }, { "speaker": "guest1", "text": "Well, one of the" }, { "speaker": "guest1", "text": "things that's kind of interesting about things that's kind of interesting about things that's kind of interesting about" }, { "speaker": "guest1", "text": "uh each model is that they each seem to uh each model is that they each seem to uh each model is that they each seem to" }, { "speaker": "guest2", "text": "have their own personality" }, { "speaker": "guest2", "text": "And one of have their own personality" }, { "speaker": "guest2", "text": "And one of have their own personality" }, { "speaker": "guest3", "text": "the things the founders are really the things the founders are really the things the founders are really" }, { "speaker": "guest3", "text": "realizing is that you're going to go to realizing is that you're going to go to realizing is that you're going to go to" }, { "speaker": "guest3", "text": "different people for different things" }, { "speaker": "host", "text": "different people for different things" }, { "speaker": "host", "text": "different people for different things" }, { "speaker": "host", "text": "Actually, one of the things that's known Actually, one of the things that's known Actually, one of the things that's known" }, { "speaker": "guest1", "text": "a lot is Claude is sort of the more a lot is Claude is sort of the more a lot is Claude is sort of the more" }, { "speaker": "guest1", "text": "happy and more human steerable model" }, { "speaker": "guest1", "text": "happy and more human steerable model" }, { "speaker": "guest2", "text": "happy and more human steerable model" }, { "speaker": "guest2", "text": "And the uh other one is Lama 4 is one And the uh other one is Lama 4 is one And the uh other one is Lama 4 is one" }, { "speaker": "guest2", "text": "that needs a lot more steering" }, { "speaker": "guest3", "text": "It's that needs a lot more steering" }, { "speaker": "guest3", "text": "It's that needs a lot more steering" }, { "speaker": "guest3", "text": "almost like talking to a developer and almost like talking to a developer and almost like talking to a developer and" }, { "speaker": "host", "text": "part of it could be an artifact of not part of it could be an artifact of not part of it could be an artifact of not" }, { "speaker": "host", "text": "having done as much RL RHF on top of it" }, { "speaker": "host", "text": "having done as much RL RHF on top of it" }, { "speaker": "guest1", "text": "having done as much RL RHF on top of it" }, { "speaker": "guest1", "text": "So is a bit more rough to work with, but So is a bit more rough to work with, but So is a bit more rough to work with, but" }, { "speaker": "guest1", "text": "you could actually steer it very well if you could actually steer it very well if you could actually steer it very well if" }, { "speaker": "guest2", "text": "you you you" }, { "speaker": "guest2", "text": "actually are good at actually doing a actually are good at actually doing a actually are good at actually doing a" }, { "speaker": "guest2", "text": "lot of prompting and almost doing a bit lot of prompting and almost doing a bit lot of prompting and almost doing a bit" }, { "speaker": "guest3", "text": "more RLHF, but it's a bit harder to work more RLHF, but it's a bit harder to work more RLHF, but it's a bit harder to work" }, { "speaker": "guest3", "text": "with actually" }, { "speaker": "guest3", "text": "Well, one of the things with actually" }, { "speaker": "host", "text": "Well, one of the things with actually" }, { "speaker": "host", "text": "Well, one of the things" }, { "speaker": "host", "text": "we've been using uh LLMs for internally we've been using uh LLMs for internally we've been using uh LLMs for internally" }, { "speaker": "guest1", "text": "is actually helping founders figure out is actually helping founders figure out is actually helping founders figure out" }, { "speaker": "guest1", "text": "who they should take money from" }, { "speaker": "guest1", "text": "And so who they should take money from" }, { "speaker": "guest2", "text": "And so who they should take money from" }, { "speaker": "guest2", "text": "in that case, sometimes you need a very in that case, sometimes you need a very in that case, sometimes you need a very" }, { "speaker": "guest2", "text": "straightforward rubric, a zero to 100" }, { "speaker": "guest3", "text": "straightforward rubric, a zero to 100" }, { "speaker": "guest3", "text": "straightforward rubric, a zero to 100" }, { "speaker": "guest3", "text": "zero being never ever take their money zero being never ever take their money zero being never ever take their money" }, { "speaker": "host", "text": "and 100 being take their money right and 100 being take their money right and 100 being take their money right" }, { "speaker": "host", "text": "Like they actually help you so away" }, { "speaker": "host", "text": "Like they actually help you so away" }, { "speaker": "guest1", "text": "Like they actually help you so" }, { "speaker": "guest1", "text": "much that you'd be crazy not to take much that you'd be crazy not to take much that you'd be crazy not to take" }, { "speaker": "guest1", "text": "their money" }, { "speaker": "guest2", "text": "Harj, we've been working on their money" }, { "speaker": "guest2", "text": "Harj, we've been working on their money" }, { "speaker": "guest2", "text": "Harj, we've been working on" }, { "speaker": "guest3", "text": "uh some scoring rubrics around that uh some scoring rubrics around that uh some scoring rubrics around that" }, { "speaker": "guest3", "text": "using prompts" }, { "speaker": "guest3", "text": "What What are some of the using prompts" }, { "speaker": "host", "text": "What What are some of the using prompts" }, { "speaker": "host", "text": "What What are some of the" }, { "speaker": "host", "text": "things we've learned" }, { "speaker": "guest1", "text": "So, it's certainly things we've learned" }, { "speaker": "guest1", "text": "So, it's certainly things we've learned" }, { "speaker": "guest1", "text": "So, it's certainly" }, { "speaker": "guest2", "text": "best practice to give um LLM's rubrics, best practice to give um LLM's rubrics, best practice to give um LLM's rubrics," }, { "speaker": "guest2", "text": "especially if you want to get a especially if you want to get a especially if you want to get a" }, { "speaker": "guest2", "text": "numerical score as the output" }, { "speaker": "guest3", "text": "You want numerical score as the output" }, { "speaker": "guest3", "text": "You want numerical score as the output" }, { "speaker": "guest3", "text": "to give it a rubric to help it to give it a rubric to help it to give it a rubric to help it" }, { "speaker": "host", "text": "understand like how should I think understand like how should I think understand like how should I think" }, { "speaker": "host", "text": "through and what's like a 80 versus a through and what's like a 80 versus a through and what's like a 80 versus a" }, { "speaker": "host", "text": "But these rubrics are never perfect" }, { "speaker": "guest1", "text": "But these rubrics are never perfect" }, { "speaker": "guest1", "text": "But these rubrics are never perfect" }, { "speaker": "guest1", "text": "there's often always exceptions and you there's often always exceptions and you there's often always exceptions and you" }, { "speaker": "guest2", "text": "tried it with uh 03 versus Gemini 2" }, { "speaker": "guest2", "text": "5 tried it with uh 03 versus Gemini 2" }, { "speaker": "guest2", "text": "5 tried it with uh 03 versus Gemini 2" }, { "speaker": "guest3", "text": "and you found this this is what we found and you found this this is what we found and you found this this is what we found" }, { "speaker": "guest3", "text": "really interesting is that um you can really interesting is that um you can really interesting is that um you can" }, { "speaker": "guest3", "text": "give the same rubric to two different give the same rubric to two different give the same rubric to two different" }, { "speaker": "host", "text": "models and in our in our specific case models and in our in our specific case models and in our in our specific case" }, { "speaker": "host", "text": "what we found is that um 03 was very what we found is that um 03 was very what we found is that um 03 was very" }, { "speaker": "host", "text": "rigid actually like it really sticks to rigid actually like it really sticks to rigid actually like it really sticks to" }, { "speaker": "guest1", "text": "the rubric it's heavily penalizes for the rubric it's heavily penalizes for the rubric it's heavily penalizes for" }, { "speaker": "guest1", "text": "anything that doesn't fit like the anything that doesn't fit like the anything that doesn't fit like the" }, { "speaker": "guest1", "text": "rubric that you've given it whereas rubric that you've given it whereas rubric that you've given it whereas" }, { "speaker": "guest2", "text": "5 Pro was actually quite good Gemini 2" }, { "speaker": "guest2", "text": "5 Pro was actually quite good Gemini 2" }, { "speaker": "guest2", "text": "5 Pro was actually quite good" }, { "speaker": "guest3", "text": "at being flexible in that it would apply at being flexible in that it would apply at being flexible in that it would apply" }, { "speaker": "guest3", "text": "the rubric but it could also sort of the rubric but it could also sort of the rubric but it could also sort of" }, { "speaker": "guest3", "text": "almost reason through why someone might almost reason through why someone might almost reason through why someone might" }, { "speaker": "host", "text": "be like an exception or why you might be like an exception or why you might be like an exception or why you might" }, { "speaker": "host", "text": "want to um push something up more want to um push something up more want to um push something up more" }, { "speaker": "host", "text": "positively or negatively than the rubric positively or negatively than the rubric positively or negatively than the rubric" }, { "speaker": "guest1", "text": "might suggest, which I just thought was might suggest, which I just thought was might suggest, which I just thought was" }, { "speaker": "guest1", "text": "really interesting cuz that it's just really interesting cuz that it's just really interesting cuz that it's just" }, { "speaker": "guest1", "text": "like when you're training a person, like when you're training a person, like when you're training a person," }, { "speaker": "guest2", "text": "you're trying to you give them a rubric you're trying to you give them a rubric you're trying to you give them a rubric" }, { "speaker": "guest2", "text": "like you want them to use a rubric as a like you want them to use a rubric as a like you want them to use a rubric as a" }, { "speaker": "guest2", "text": "guide, but there are always these sort guide, but there are always these sort guide, but there are always these sort" }, { "speaker": "guest3", "text": "of edge cases where you need to sort of of edge cases where you need to sort of of edge cases where you need to sort of" }, { "speaker": "guest3", "text": "think a little bit more deeply" }, { "speaker": "guest3", "text": "Um, and think a little bit more deeply" }, { "speaker": "host", "text": "Um, and think a little bit more deeply" }, { "speaker": "host", "text": "I just thought it was interesting that I just thought it was interesting that I just thought it was interesting that" }, { "speaker": "host", "text": "the models themselves will handle that the models themselves will handle that the models themselves will handle that" }, { "speaker": "guest1", "text": "differently, which means they sort of differently, which means they sort of differently, which means they sort of" }, { "speaker": "guest1", "text": "have different personalities, right" }, { "speaker": "guest1", "text": "have different personalities, right" }, { "speaker": "guest2", "text": "have different personalities, right" }, { "speaker": "guest2", "text": "Like 03 felt a little bit more like the Like 03 felt a little bit more like the Like 03 felt a little bit more like the" }, { "speaker": "guest2", "text": "soldier sort of like, okay, I'm soldier sort of like, okay, I'm soldier sort of like, okay, I'm" }, { "speaker": "guest3", "text": "definitely like check, check, check, definitely like check, check, check, definitely like check, check, check," }, { "speaker": "guest3", "text": "check, check" }, { "speaker": "guest3", "text": "Um, and Gemini Pro 2" }, { "speaker": "host", "text": "5 check, check" }, { "speaker": "host", "text": "Um, and Gemini Pro 2" }, { "speaker": "host", "text": "5 check, check" }, { "speaker": "guest1", "text": "Um, and Gemini Pro 2" }, { "speaker": "guest1", "text": "felt a little bit more like a a high felt a little bit more like a a high felt a little bit more like a a high" }, { "speaker": "guest1", "text": "agency sort of employee was like, \"Oh, agency sort of employee was like, \"Oh, agency sort of employee was like, \"Oh," }, { "speaker": "guest2", "text": "I think this makes sense, but this okay" }, { "speaker": "guest2", "text": "I think this makes sense, but this okay" }, { "speaker": "guest2", "text": "I think this makes sense, but this" }, { "speaker": "guest3", "text": "might be an exception in this case,\" might be an exception in this case,\" might be an exception in this case,\"" }, { "speaker": "guest3", "text": "which was um just really interesting to which was um just really interesting to which was um just really interesting to" }, { "speaker": "guest3", "text": "Yeah, it's funny to see that for see" }, { "speaker": "host", "text": "Yeah, it's funny to see that for see" }, { "speaker": "host", "text": "Yeah, it's funny to see that for" }, { "speaker": "host", "text": "You know, sometimes you have investors" }, { "speaker": "guest1", "text": "You know, sometimes you have investors" }, { "speaker": "guest1", "text": "You know, sometimes you have" }, { "speaker": "guest1", "text": "investors like a Benchmark or a Thrive, investors like a Benchmark or a Thrive, investors like a Benchmark or a Thrive," }, { "speaker": "guest2", "text": "it's like, \"Yeah, take their money right it's like, \"Yeah, take their money right it's like, \"Yeah, take their money right" }, { "speaker": "guest2", "text": "Their process is immaculate" }, { "speaker": "guest2", "text": "Their process is immaculate" }, { "speaker": "guest3", "text": "Their process is immaculate" }, { "speaker": "guest3", "text": "never ghost anyone" }, { "speaker": "guest3", "text": "They answer their never ghost anyone" }, { "speaker": "host", "text": "They answer their never ghost anyone" }, { "speaker": "host", "text": "They answer their" }, { "speaker": "host", "text": "emails faster than most founders" }, { "speaker": "guest1", "text": "It's, emails faster than most founders" }, { "speaker": "guest1", "text": "It's, emails faster than most founders" }, { "speaker": "guest1", "text": "you know, very impressive" }, { "speaker": "guest2", "text": "And then, uh, you know, very impressive" }, { "speaker": "guest2", "text": "And then, uh, you know, very impressive" }, { "speaker": "guest2", "text": "And then, uh," }, { "speaker": "guest3", "text": "one example here might be, you know, one example here might be, you know, one example here might be, you know," }, { "speaker": "guest3", "text": "there are plenty of investors who are there are plenty of investors who are there are plenty of investors who are" }, { "speaker": "guest3", "text": "just overwhelmed and maybe they're just just overwhelmed and maybe they're just just overwhelmed and maybe they're just" }, { "speaker": "host", "text": "not that good at managing their time" }, { "speaker": "host", "text": "not that good at managing their time" }, { "speaker": "host", "text": "not that good at managing their time" }, { "speaker": "guest1", "text": "And so, they might be really great And so, they might be really great And so, they might be really great" }, { "speaker": "guest1", "text": "investors and their track record bears investors and their track record bears investors and their track record bears" }, { "speaker": "guest1", "text": "that out, but they're sort of slow to that out, but they're sort of slow to that out, but they're sort of slow to" }, { "speaker": "guest2", "text": "They seem overwhelmed all the get back" }, { "speaker": "guest2", "text": "They seem overwhelmed all the get back" }, { "speaker": "guest2", "text": "They seem overwhelmed all the" }, { "speaker": "guest3", "text": "They accidentally, probably not time" }, { "speaker": "guest3", "text": "They accidentally, probably not time" }, { "speaker": "guest3", "text": "They accidentally, probably not" }, { "speaker": "host", "text": "intentionally ghost people" }, { "speaker": "host", "text": "And so this intentionally ghost people" }, { "speaker": "host", "text": "And so this intentionally ghost people" }, { "speaker": "guest1", "text": "And so this" }, { "speaker": "guest1", "text": "is legitimately exactly what an LLM is is legitimately exactly what an LLM is is legitimately exactly what an LLM is" }, { "speaker": "guest1", "text": "Like the debug info on some of for" }, { "speaker": "guest2", "text": "Like the debug info on some of for" }, { "speaker": "guest2", "text": "Like the debug info on some of" }, { "speaker": "guest2", "text": "these are very interesting to see like these are very interesting to see like these are very interesting to see like" }, { "speaker": "guest3", "text": "you know maybe it's a 91 instead of like you know maybe it's a 91 instead of like you know maybe it's a 91 instead of like" }, { "speaker": "guest3", "text": "I guess one of the an 89" }, { "speaker": "guest3", "text": "I guess one of the an 89" }, { "speaker": "host", "text": "I guess one of the" }, { "speaker": "host", "text": "things that's been really surprising to things that's been really surprising to things that's been really surprising to" }, { "speaker": "host", "text": "me as you know we ourselves are playing me as you know we ourselves are playing me as you know we ourselves are playing" }, { "speaker": "guest1", "text": "with it and we spend you know maybe 80 with it and we spend you know maybe 80 with it and we spend you know maybe 80" }, { "speaker": "guest1", "text": "to 90% of our time with founders who are to 90% of our time with founders who are to 90% of our time with founders who are" }, { "speaker": "guest1", "text": "all the way out on the edge is uh you all the way out on the edge is uh you all the way out on the edge is uh you" }, { "speaker": "guest2", "text": "know on the one hand the analogies I know on the one hand the analogies I know on the one hand the analogies I" }, { "speaker": "guest2", "text": "think even we use to discuss this is uh think even we use to discuss this is uh think even we use to discuss this is uh" }, { "speaker": "guest2", "text": "it's kind of like coding" }, { "speaker": "guest3", "text": "It kind of it's kind of like coding" }, { "speaker": "guest3", "text": "It kind of it's kind of like coding" }, { "speaker": "guest3", "text": "actually feels like coding in, you know, actually feels like coding in, you know, actually feels like coding in, you know," }, { "speaker": "host", "text": "Like the tools are not all the way 1995" }, { "speaker": "host", "text": "Like the tools are not all the way 1995" }, { "speaker": "host", "text": "Like the tools are not all the way" }, { "speaker": "guest1", "text": "There's a lot of stuff that's there" }, { "speaker": "guest1", "text": "There's a lot of stuff that's there" }, { "speaker": "guest1", "text": "There's a lot of stuff that's" }, { "speaker": "guest2", "text": "unspecified" }, { "speaker": "guest2", "text": "We're, you know, in this unspecified" }, { "speaker": "guest2", "text": "We're, you know, in this unspecified" }, { "speaker": "guest3", "text": "We're, you know, in this" }, { "speaker": "guest3", "text": "new frontier" }, { "speaker": "guest3", "text": "But personally, it also new frontier" }, { "speaker": "host", "text": "But personally, it also new frontier" }, { "speaker": "host", "text": "But personally, it also" }, { "speaker": "host", "text": "kind of feels like learning how to kind of feels like learning how to kind of feels like learning how to" }, { "speaker": "guest1", "text": "manage a person where it's like, how do manage a person where it's like, how do manage a person where it's like, how do" }, { "speaker": "guest1", "text": "I actually communicate uh, you know, the I actually communicate uh, you know, the I actually communicate uh, you know, the" }, { "speaker": "guest1", "text": "things that they need to know in order things that they need to know in order things that they need to know in order" }, { "speaker": "guest2", "text": "to make a good decision" }, { "speaker": "guest2", "text": "And how do I to make a good decision" }, { "speaker": "guest2", "text": "And how do I to make a good decision" }, { "speaker": "guest3", "text": "And how do I" }, { "speaker": "guest3", "text": "make sure that they know um, you know, make sure that they know um, you know, make sure that they know um, you know," }, { "speaker": "guest3", "text": "how I'm going to evaluate and score how I'm going to evaluate and score how I'm going to evaluate and score" }, { "speaker": "host", "text": "And uh not only that, like there's them" }, { "speaker": "host", "text": "And uh not only that, like there's them" }, { "speaker": "host", "text": "And uh not only that, like there's" }, { "speaker": "guest1", "text": "this aspect of Kaizen, you know, this um this aspect of Kaizen, you know, this um this aspect of Kaizen, you know, this um" }, { "speaker": "guest1", "text": "this manufacturing technique that this manufacturing technique that this manufacturing technique that" }, { "speaker": "guest1", "text": "created really really good cars for created really really good cars for created really really good cars for" }, { "speaker": "guest2", "text": "Japan in the '90s" }, { "speaker": "guest2", "text": "Uh and that principle Japan in the '90s" }, { "speaker": "guest2", "text": "Uh and that principle Japan in the '90s" }, { "speaker": "guest3", "text": "Uh and that principle" }, { "speaker": "guest3", "text": "actually says that the people who are actually says that the people who are actually says that the people who are" }, { "speaker": "guest3", "text": "the absolute best at improving the the absolute best at improving the the absolute best at improving the" }, { "speaker": "host", "text": "process are the people actually doing process are the people actually doing process are the people actually doing" }, { "speaker": "host", "text": "That's literally why uh Japanese it" }, { "speaker": "host", "text": "That's literally why uh Japanese it" }, { "speaker": "guest1", "text": "That's literally why uh Japanese" }, { "speaker": "guest1", "text": "cars got so good in the '90s" }, { "speaker": "guest1", "text": "And that's cars got so good in the '90s" }, { "speaker": "guest2", "text": "And that's cars got so good in the '90s" }, { "speaker": "guest2", "text": "metaprompting to me" }, { "speaker": "guest2", "text": "So, I don't know" }, { "speaker": "guest3", "text": "metaprompting to me" }, { "speaker": "guest3", "text": "So, I don't know" }, { "speaker": "guest3", "text": "metaprompting to me" }, { "speaker": "host", "text": "So, I don't know" }, { "speaker": "host", "text": "It's a brave new world" }, { "speaker": "host", "text": "We're sort of in It's a brave new world" }, { "speaker": "guest1", "text": "We're sort of in It's a brave new world" }, { "speaker": "guest1", "text": "We're sort of in" }, { "speaker": "guest1", "text": "this new moment" }, { "speaker": "guest2", "text": "So, with that, we're this new moment" }, { "speaker": "guest2", "text": "So, with that, we're this new moment" }, { "speaker": "guest2", "text": "So, with that, we're" }, { "speaker": "guest3", "text": "out of time" }, { "speaker": "guest3", "text": "But can't wait to see what out of time" }, { "speaker": "guest3", "text": "But can't wait to see what out of time" }, { "speaker": "host", "text": "But can't wait to see what" }, { "speaker": "host", "text": "kind of prompts you guys come up with" }, { "speaker": "host", "text": "kind of prompts you guys come up with" }, { "speaker": "guest1", "text": "kind of prompts you guys come up with" }, { "speaker": "guest1", "text": "And we'll see you next time" }, { "speaker": "guest1", "text": "And we'll see you next time" }, { "speaker": "guest2", "text": "And we'll see you next time" } ]