Documentation ¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
View Source
var ConvertCmd = &cobra.Command{ Use: "convert", Aliases: []string{"c"}, Short: "convert netaffiliation catalog to golang structs.", Long: "convert netaffiliation catalog to golang structs", Run: func(cmd *cobra.Command, args []string) { client := grab.NewClient() catalogURL := "https://flux.netaffiliation.com/feed.php?lstv3=CD9B4220P1045983100L44D2F" tr := &http.Transport{ DialContext: (&net.Dialer{ Timeout: 240 * time.Second, DualStack: true, }).DialContext, MaxIdleConns: 100, IdleConnTimeout: 240 * time.Second, TLSHandshakeTimeout: 240 * time.Second, ExpectContinueTimeout: 1 * time.Second, DisableKeepAlives: true, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, } client.UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36" client.HTTPClient = &http.Client{ Timeout: 240 * time.Second, Transport: tr, CheckRedirect: func(req *http.Request, via []*http.Request) error { return nil }, } catalogXmlFile := filepath.Join("catalogs", "netaffiliation.xml") if _, err := os.Stat(catalogXmlFile); os.IsNotExist(err) { req, err := grab.NewRequest(catalogXmlFile, catalogURL) if err != nil { log.Fatal(err) } fmt.Printf("Downloading %v...\n", req.URL()) resp := client.Do(req) if resp.HTTPResponse == nil { log.Fatal(err) } fmt.Printf(" %v\n", resp.HTTPResponse.Status) t := time.NewTicker(500 * time.Millisecond) defer t.Stop() LoopRel: for { select { case <-t.C: fmt.Printf(" transferred %v / %v bytes (%.2f%%)\n", resp.BytesComplete(), resp.Size, 100*resp.Progress()) case <-resp.Done: break LoopRel } } if err := resp.Err(); err != nil { log.Fatalln("Download failed:", err, catalogURL) } fmt.Printf("Download saved to %v \n", resp.Filename) catalogXmlFile = resp.Filename } file, err := os.Open(catalogXmlFile) if err != nil { log.Fatal(err) } defer file.Close() root := new(zek.Node) root.MaxExamples = maxExamples reader := bufio.NewReader(file) if _, err := root.ReadFrom(reader); err != nil { log.Fatalf("could not read '%s', error: %s", catalogXmlFile, err) } if tagName != "" { if n := root.ByName(tagName); n != nil { root = n } } root.Name = xml.Name{Space: "", Local: "feeds"} var buf bytes.Buffer io.WriteString(&buf, tmplPackage) sw := zek.NewStructWriter(&buf) sw.WithComments = withComments sw.WithJSONTags = withJSONTags sw.Strict = strict sw.ExampleMaxChars = exampleMaxChars sw.Compact = !nonCompact sw.UniqueExamples = uniqueExamples sw.OmitEmptyText = omitEmptyText if err := sw.WriteNode(root); err != nil { log.Fatal(err) } outputFile := filepath.Join(outputDir, fmt.Sprintf("%s.go", "feeds")) pp.Println("outputFile", outputFile) f, err := os.Create(outputFile) if err != nil { log.Fatal(err) } defer f.Close() if !skipFormatting { b, err := format.Source(buf.Bytes()) if err != nil { log.Fatal(err) } _, err = f.Write(b) if err != nil { log.Fatal(err) } } else { _, err := f.WriteString(buf.String()) if err != nil { log.Fatal(err) } pp.Println(buf.String()) } }, }
View Source
var PrepareCmd = &cobra.Command{ Use: "prepare", Aliases: []string{"p"}, Short: "prepare csv catalog to prestashop import format (core or webkul's marketplace).", Long: "prepare csv catalog to prestashop import format (core or webkul's marketplace).", Run: func(cmd *cobra.Command, args []string) { gtotal := 0 if !dryRun { var err error dsn := fmt.Sprintf("%v:%v@tcp(%v:%v)/%v?charset=utf8mb4&collation=utf8mb4_general_ci&parseTime=True&loc=Local", dbUser, dbPass, dbHost, dbPort, dbName) db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{ NamingStrategy: schema.NamingStrategy{ TablePrefix: dbTablePrefix, SingularTable: true, NameReplacer: strings.NewReplacer("ID", "Id"), }, }) if err != nil { log.Fatal(err) } } var err error proxyURL, err = url.Parse(proxyURLStr) if err != nil { log.Fatal(err) } kv, err = badger.Open(badger.DefaultOptions(kvPath)) if err != nil { log.Fatal(err) } defer kv.Close() osPathname := "catalogs/netaffiliation.xml" if options.debug { pp.Println("osPathname:", osPathname) } doc := etree.NewDocument() if err := doc.ReadFromFile(osPathname); err != nil { panic(err) } root := doc.SelectElement("campaigns") fmt.Println("ROOT element:", root.Tag) var catalogs []string reStruct := regexp.MustCompile(`type (.*) struct`) reTags := regexp.MustCompile(`json:"(.*)"`) err = db.Where("active = ?", 1).Find(&activeLangs).Error pp.Println("activeLangs:", activeLangs) err = db.Where("active = ?", 1).Find(&activeShops).Error if errors.Is(err, gorm.ErrRecordNotFound) { log.Fatal("active shops not found") } pp.Println("activeShops:", activeShops) err = db.Find(&activeGroups).Error if errors.Is(err, gorm.ErrRecordNotFound) { log.Fatal("groups not found") } pp.Println("activeGroups:", activeGroups) campaigns := root.SelectElements("campaign") t := throttler.New(3, len(campaigns)) for _, campaign := range campaigns { go func(c *etree.Element) error { defer t.Done(nil) var feedURL string var feedName string var feedVersion string for _, pf := range c.SelectElements("product_feeds") { if product_feed_url := pf.SelectElement("product_feed"); product_feed_url != nil { feedVersion = product_feed_url.SelectAttrValue("version", "unknown") if feedVersion != "4" { continue } feedURL = product_feed_url.Text() feedName = product_feed_url.SelectAttrValue("name", "unknown") } } pp.Println("feedName:", feedName) pp.Println("feedURL:", feedURL) pp.Println("feedVersion:", feedVersion) cmp := &CatalogMap{ Name: feedName, Feed: feedURL, } if psMarketplace { var randomCountry psm.Country db.Order("RAND()").First(&randomCountry) var randomLanguage psm.Lang db.Order("RAND()").First(&randomLanguage) var randomCustomer psm.Customer db.Order("RAND()").First(&randomCustomer) if feedName == "" { feedName = gofakeit.Company() } seller := WkMpSeller{ ShopNameUnique: feedName, LinkRewrite: slug.Make(feedName), SellerFirstname: gofakeit.FirstName(), SellerLastname: gofakeit.LastName(), BusinessEmail: gofakeit.Email(), Phone: gofakeit.Phone(), Fax: "", Address: gofakeit.Street(), Postcode: gofakeit.Zip(), City: gofakeit.City(), IdCountry: int(randomCountry.IDCountry), IdState: 0, TaxIdentificationNumber: gofakeit.UUID(), DefaultLang: int(randomLanguage.IDLang), FacebookId: "", TwitterId: "", GoogleId: "", InstagramId: "", ProfileImage: "", ProfileBanner: "", ShopImage: "", ShopBanner: "", Active: true, ShopApproved: true, SellerCustomerId: int(randomCustomer.IDCustomer), SellerDetailsAccess: "", DateAdd: time.Now(), DateUpd: time.Now(), } var wkMpSeller WkMpSeller err := db.Where("shop_name_unique = ? ", feedName).First(&wkMpSeller).Error if errors.Is(err, gorm.ErrRecordNotFound) { err := db.Create(&seller).Error if err != nil { log.Fatal(err) } db.Where("shop_name_unique = ? ", feedName).First(&wkMpSeller) cmp.SellerId = wkMpSeller.IdSeller for _, activeLang := range activeLangs { wkMpSellerLang := &WkMpSellerLang{ IdSeller: wkMpSeller.IdSeller, IdLang: activeLang.IDLang, ShopName: feedName, AboutShop: gofakeit.HackerPhrase(), } err := db.Create(&wkMpSellerLang).Error if err != nil { return err } } } else { cmp.SellerId = wkMpSeller.IdSeller } } cmp.Mapping.LangSuffix = languagesDef cmp.Mapping.LangFields = []string{"name", "description", "description_short"} client := grab.NewClient() tr := &http.Transport{ DialContext: (&net.Dialer{ Timeout: 240 * time.Second, DualStack: true, }).DialContext, MaxIdleConns: 100, IdleConnTimeout: 240 * time.Second, TLSHandshakeTimeout: 240 * time.Second, ExpectContinueTimeout: 1 * time.Second, DisableKeepAlives: true, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, } client.UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36" client.HTTPClient = &http.Client{ Timeout: 240 * time.Second, Transport: tr, CheckRedirect: func(req *http.Request, via []*http.Request) error { return nil }, } feedName = stripCtlAndExtFromUnicode(feedName) localCatalogStruct := filepath.Join("..", "..", "internal", "netaffiliation", fmt.Sprintf("%s.go", strcase.ToSnake(feedName))) localCatalogJson := filepath.Join("catalogs", fmt.Sprintf("%s.json", slug.Make(feedName))) localCatalogCsv := filepath.Join("catalogs", fmt.Sprintf("%s.csv", slug.Make(feedName))) localCatalogMapYaml := filepath.Join("imports", fmt.Sprintf("%s.yml", slug.Make(feedName))) localCatalogFormattedCsv := filepath.Join("imports", fmt.Sprintf("%s.csv", slug.Make(feedName))) if _, err := os.Stat(localCatalogCsv); os.IsNotExist(err) { req, err := grab.NewRequest(localCatalogCsv, feedURL) if err != nil { log.Warnln(err) return err } fmt.Printf("Downloading %v...\n", req.URL()) resp := client.Do(req) if resp.HTTPResponse == nil { log.Warnln(err) return err } fmt.Printf(" %v\n", resp.HTTPResponse.Status) t := time.NewTicker(500 * time.Millisecond) defer t.Stop() LoopRel: for { select { case <-t.C: fmt.Printf(" transferred %v / %v bytes (%.2f%%)\n", resp.BytesComplete(), resp.Size, 100*resp.Progress()) case <-resp.Done: break LoopRel } } if err := resp.Err(); err != nil { log.Warnln("Download failed:", err, feedURL) return err } fmt.Printf("Download saved to %v \n", resp.Filename) localCatalogCsv = resp.Filename } inputFile, err := os.Open(localCatalogCsv) if err != nil { log.Warnln(err) return err } defer inputFile.Close() detect := detector.New() delimiters := detect.DetectDelimiter(inputFile, '"') if len(delimiters) == 0 || len(delimiters) > 1 { return errors.New("No delimiter detected") } cmp.Separator = delimiters[0] cols, err := getHeaders(localCatalogCsv, delimiters[0]) if err != nil { log.Warnln(err) return err } cmp.Fields = cols cmp.Mapping.Update = true for _, col := range cols { switch col { case "url", "product_url", "ur_lproduit", "product_page_url", "link", "url_de_la_page_produit", "url_produit", "prod_url", "product_link": cmp.Mapping.Product.Redirect = col case "name", "nom", "titre", "name_of_the_product", "title", "product_name", "prod_name", "nom_attribute", "nom_usuel_du_produit": cmp.Mapping.Product.Name = col case "reference", "ref", "reference_interne", "identifiant", "internal_reference", "reference_du_produit", "reference_fabriquant": cmp.Mapping.Product.Reference = col case "ean13", "ean", "ean_or_isbn", "ean_13", "reference_universelle", "universal_reference", "prod_ean", "id", "code_ean": cmp.Mapping.Product.Ean13 = col case "sku", "numero_modele_produit": cmp.Mapping.Product.Sku = col case "prix", "price", "current_price", "prix_de_vente", "prix_actuel", "prod_price", "prix_ttc", "prix_ttc_du_produit": cmp.Mapping.Product.Price = col case "mpn": cmp.Mapping.Product.Mpn = col case "description", "descriptif", "product_detail", "description_html", "discription_of_the_product", "prod_description_long": cmp.Mapping.Product.Description = col case "description_short", "product_highlight", "prod_description": cmp.Mapping.Product.DescriptionShort = col case "image": cmp.Mapping.Product.Image = col case "quantity", "quantite", "stock", "indicateur_de_stock", "stock_indicator", "qty": cmp.Mapping.Product.Quantity = col case "img_large", "url_grande", "url_image_grande", "url_image", "big_image", "url_de_l'image_par_defaut", "image_link", "image_product", "url_of_the_big_image", "image_default", "image_url_1", "image_url": cmp.Mapping.Product.Image = col case "hauteur_cm": cmp.Mapping.Product.Height = col case "largeur_cm": cmp.Mapping.Product.Width = col case "poids_kg", "weight": cmp.Mapping.Product.Weight = col case "shipping_and_handling_cost", "frais_de_port", "frais_de_ports", "shipping": cmp.Mapping.Product.Shipping = col case "taille", "couleur", "matiere", "size", "color": cmp.Mapping.Product.Attributes = append(cmp.Mapping.Product.Attributes, col) case "marque", "brand", "genre", "garantie", "modele", "saison", "collection", "fabricant", "condition", "name_of_brand", "montage", "nom_marque", "type_peinture", "brand_name", "famille_de_produit", "largeur", "hauteur", "vitesse", "diametre", "charge", "runflat", "vehicule", "rechape": cmp.Mapping.Product.Features = append(cmp.Mapping.Product.Features, col) case "categorie_finale": cmp.Mapping.Category.Name = col case "category_breadcrumb", "categorie", "category", "breadcrumb", "product_category", "noms_de_toutes_les_categories": cmp.Mapping.Category.Breadcrumb = col } } cmp, total, err := csv2ps(db, localCatalogCsv, localCatalogFormattedCsv, cols, delimiters[0], cmp) checkErr("while writing formatted for prestashop csv file", err) cmp.Total = total gtotal += total outputBytes, err := csv2json(localCatalogCsv, cols, delimiters[0]) if err != nil { log.Warnln(err) return err } cmpBytes, err := yaml.Marshal(cmp) if err != nil { fmt.Printf("err: %v\n", err) return err } err = ioutil.WriteFile(localCatalogMapYaml, cmpBytes, 0644) checkErr("while writing mapping yaml file", err) err = ioutil.WriteFile(localCatalogJson, outputBytes, 0644) checkErr("while writing json file", err) json2struct.SetDebug(options.debug) opt := json2struct.Options{ UseOmitempty: false, UseShortStruct: true, UseLocal: false, UseExample: false, Prefix: "", Suffix: "", Name: strings.ToLower(feedName), } jsonFile, err := os.Open(localCatalogJson) if err != nil { log.Warnln(err) return err } defer jsonFile.Close() parsed, err := json2struct.Parse(jsonFile, opt) if err != nil { return err } csvRowLine := 1 csvRowExample, err := getExampleRow(localCatalogCsv, delimiters[0], cols, csvRowLine) if err != nil { return err } structName := reStruct.FindStringSubmatch(parsed) parsed = reTags.ReplaceAllString(parsed, `json:"$1" struct2map:"key:$1"`) catalogs = append(catalogs, structName[1]) structResult := bytes.NewBufferString("") structTemplate, _ := template.New("").Parse(packageTemplate) structTemplate.Execute(structResult, map[string]string{ "CatalogSeparator": delimiters[0], "CatalogPath": localCatalogCsv, "Line": fmt.Sprintf("%d", csvRowLine), "Struct": parsed, "StructName": stripCtlAndExtFromUnicode(structName[1]), "Row": strings.Join(csvRowExample, ",\n")}, ) err = ioutil.WriteFile(localCatalogStruct, structResult.Bytes(), 0644) if err != nil { return err } return nil }(campaign) t.Throttle() } if t.Err() != nil { for i, err := range t.Errs() { log.Printf("error #%d: %s", i, err) } log.Fatal(t.Err()) } localCatalogBase := filepath.Join("..", "..", "internal", "netaffiliation", fmt.Sprintf("%s.go", "base")) catalogResult := bytes.NewBufferString("") catalogTemplate, _ := template.New("").Parse(baseTemplate) catalogTemplate.Execute(catalogResult, map[string][]string{"Catalogs": catalogs}) err = ioutil.WriteFile(localCatalogBase, catalogResult.Bytes(), 0644) checkErr("while writing struct file", err) pp.Println("total new entries:", gtotal) }, }
View Source
var RootCmd = &cobra.Command{
Use: "netaf2ps",
Short: "netaf2ps is an helper to load csv netaffiliation catalogs into a prestashop database.",
Long: `netaf2ps is an helper to load csv netaffiliation catalogs into a prestashop database.`,
}
RootCmd is the root command for ovh-qa
Functions ¶
func Execute ¶
func Execute()
Execute adds all child commands to the root command and sets flags appropriately.
func ReplaceSoloCarriageReturns ¶
ReplaceSoloCarriageReturns wraps an io.Reader, on every call of Read it for instances of lonely \r replacing them with \r\n before returning to the end customer lots of files in the wild will come without "proper" line breaks, which irritates go's standard csv package. This'll fix by wrapping the reader passed to csv.NewReader:
rdr, err := csv.NewReader(ReplaceSoloCarriageReturns(r))
Types ¶
type CatalogMap ¶
type CatalogMap struct { Name string `yaml:"name"` Feed string `yaml:"feed"` SellerId int `yaml:"seller_id"` Separator string `yaml:"separator"` Fields []string `yaml:"fields"` Total int `yaml:"total"` Mapping struct { Update bool `yaml:"update" default:"true"` // Either ADD LangSuffix []string `yaml:"multi_language_suffix"` LangFields []string `yaml:"multi_language_fields"` Product Product `yaml:"product"` Category struct { Name string `yaml:"name"` Breadcrumb string `yaml:"breaddcrumb"` Separator string `yaml:"separator"` } `yaml:"category"` } `yaml:"mapping"` }
type Product ¶
type Product struct { Name string `yaml:"name"` Reference string `yaml:"reference"` Ean13 string `yaml:"ean13"` Sku string `yaml:"sku"` Mpn string `yaml:"mpn"` Price string `yaml:"price"` Description string `yaml:"description"` DescriptionShort string `yaml:"description_short"` Image string `yaml:"image"` Quantity string `yaml:"quantity"` Width string `yaml:"width"` Height string `yaml:"height"` Weight string `yaml:"weight"` Shipping string `yaml:"shipping"` Redirect string `yaml:"redirect"` Attributes []string `yaml:"attributes"` Features []string `yaml:"features"` }
type WkMpSeller ¶
type WkMpSeller struct { IdSeller int ShopNameUnique string LinkRewrite string SellerFirstname string SellerLastname string BusinessEmail string Phone string Fax string Address string Postcode string City string IdCountry int IdState int TaxIdentificationNumber string DefaultLang int FacebookId string TwitterId string GoogleId string InstagramId string ProfileImage string ProfileBanner string ShopImage string ShopBanner string Active bool ShopApproved bool SellerCustomerId int SellerDetailsAccess string DateAdd time.Time DateUpd time.Time }
type WkMpSellerLang ¶
type WkMpSellerProduct ¶
type WkMpSellerProduct struct { IdMpProduct int // primary_key IdSeller int IdPsProduct int IdPsShop int IdCategory int Price float64 WholesalePrice float64 Unity string UnitPrice float64 IdTaxRulesGroup int OnSale bool AdditionalShippingCost float64 Quantity int MinimalQuantity int LowStockThreshold int LowStockAlert bool Active bool StatusBeforeDeactivate bool ShowCondition bool Condition string AvailableForOrder bool ShowPrice bool OnlineOnly bool Visibility string AdminAssigned bool Width float64 Height float64 Depth float64 Weight float64 Reference string Ean13 string Upc string Isbn string OutOfStock int AvailableDate time.Time PsIdCarrierReference string AdminApproved bool AdditionalDeliveryTimes bool DateAdd time.Time DateUpd time.Time CsvRequestNo string }
type WkMpSellerProductImage ¶
type WkMpSellerProductLang ¶
Click to show internal directories.
Click to hide internal directories.