Documentation ¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
View Source
var ImportCmd = &cobra.Command{ Use: "import", Aliases: []string{"i"}, Short: "import effiliation csv catalog to prestashop import format (core or webkul's marketplace).", Long: "import effiliation csv catalog to prestashop import format (core or webkul's marketplace).", Run: func(cmd *cobra.Command, args []string) { gtotal := 0 if !dryRun { var err error dsn := fmt.Sprintf("%v:%v@tcp(%v:%v)/%v?charset=utf8mb4&collation=utf8mb4_general_ci&parseTime=True&loc=Local", dbUser, dbPass, dbHost, dbPort, dbName) db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{ NamingStrategy: schema.NamingStrategy{ TablePrefix: dbTablePrefix, SingularTable: true, NameReplacer: strings.NewReplacer("ID", "Id"), }, }) if err != nil { log.Fatal(err) } } var err error proxyURL, err = url.Parse(proxyURLStr) if err != nil { log.Fatal(err) } kv, err = badger.Open(badger.DefaultOptions(kvPath)) if err != nil { log.Fatal(err) } defer kv.Close() osPathname := "catalogs/effiliation.xml" if options.debug { pp.Println("osPathname:", osPathname) } doc := etree.NewDocument() if err := doc.ReadFromFile(osPathname); err != nil { panic(err) } root := doc.SelectElement("feeds") fmt.Println("ROOT element:", root.Tag) var catalogs []string reStruct := regexp.MustCompile(`type (.*) struct`) reTags := regexp.MustCompile(`json:"(.*)"`) err = db.Where("active = ?", 1).Find(&activeLangs).Error pp.Println("activeLangs:", activeLangs) err = db.Where("active = ?", 1).Find(&activeShops).Error if errors.Is(err, gorm.ErrRecordNotFound) { log.Fatal("active shops not found") } pp.Println("activeShops:", activeShops) err = db.Find(&activeGroups).Error if errors.Is(err, gorm.ErrRecordNotFound) { log.Fatal("groups not found") } pp.Println("activeGroups:", activeGroups) feeds := root.SelectElements("feed") t := throttler.New(3, len(feeds)) for _, feed := range feeds { go func(f *etree.Element) error { defer t.Done(nil) var feedURL string var feedName string if product_feed_url := f.SelectElement("code"); product_feed_url != nil { feedURL = product_feed_url.Text() } if product_feed_name := f.SelectElement("nomprogramme"); product_feed_name != nil { feedName = product_feed_name.Text() } pp.Println("feedName:", feedName) pp.Println("feedURL:", feedURL) cmp := &CatalogMap{ Name: feedName, Feed: feedURL, } if psMarketplace { var randomCountry psm.Country db.Order("RAND()").First(&randomCountry) var randomLanguage psm.Lang db.Order("RAND()").First(&randomLanguage) var randomCustomer psm.Customer db.Order("RAND()").First(&randomCustomer) if feedName == "" { feedName = gofakeit.Company() } seller := WkMpSeller{ ShopNameUnique: feedName, LinkRewrite: slug.Make(feedName), SellerFirstname: gofakeit.FirstName(), SellerLastname: gofakeit.LastName(), BusinessEmail: gofakeit.Email(), Phone: gofakeit.Phone(), Fax: "", Address: gofakeit.Street(), Postcode: gofakeit.Zip(), City: gofakeit.City(), IdCountry: int(randomCountry.IDCountry), IdState: 0, TaxIdentificationNumber: gofakeit.UUID(), DefaultLang: int(randomLanguage.IDLang), FacebookId: "", TwitterId: "", GoogleId: "", InstagramId: "", ProfileImage: "", ProfileBanner: "", ShopImage: "", ShopBanner: "", Active: true, ShopApproved: true, SellerCustomerId: int(randomCustomer.IDCustomer), SellerDetailsAccess: "", DateAdd: time.Now(), DateUpd: time.Now(), } var wkMpSeller WkMpSeller err := db.Where("shop_name_unique = ? ", feedName).First(&wkMpSeller).Error if errors.Is(err, gorm.ErrRecordNotFound) { err := db.Create(&seller).Error if err != nil { log.Fatal(err) } db.Where("shop_name_unique = ? ", feedName).First(&wkMpSeller) cmp.SellerId = wkMpSeller.IdSeller for _, activeLang := range activeLangs { wkMpSellerLang := &WkMpSellerLang{ IdSeller: wkMpSeller.IdSeller, IdLang: activeLang.IDLang, ShopName: feedName, AboutShop: gofakeit.HackerPhrase(), } err := db.Create(&wkMpSellerLang).Error if err != nil { return err } } } else { cmp.SellerId = wkMpSeller.IdSeller } } cmp.Mapping.LangSuffix = languagesDef cmp.Mapping.LangFields = []string{"name", "description", "description_short"} client := grab.NewClient() tr := &http.Transport{ DialContext: (&net.Dialer{ Timeout: 240 * time.Second, DualStack: true, }).DialContext, MaxIdleConns: 100, IdleConnTimeout: 240 * time.Second, TLSHandshakeTimeout: 240 * time.Second, ExpectContinueTimeout: 1 * time.Second, DisableKeepAlives: true, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, } client.UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36" client.HTTPClient = &http.Client{ Timeout: 240 * time.Second, Transport: tr, CheckRedirect: func(req *http.Request, via []*http.Request) error { return nil }, } feedName = stripCtlAndExtFromUnicode(feedName) localCatalogStruct := filepath.Join("..", "..", "internal", "effiliation", fmt.Sprintf("%s.go", strcase.ToSnake(feedName))) localCatalogJson := filepath.Join("catalogs", fmt.Sprintf("%s.json", slug.Make(feedName))) localCatalogCsv := filepath.Join("catalogs", fmt.Sprintf("%s.csv", slug.Make(feedName))) localCatalogMapYaml := filepath.Join("imports", fmt.Sprintf("%s.yml", slug.Make(feedName))) localCatalogFormattedCsv := filepath.Join("imports", fmt.Sprintf("%s.csv", slug.Make(feedName))) if _, err := os.Stat(localCatalogCsv); os.IsNotExist(err) { req, err := grab.NewRequest(localCatalogCsv, feedURL) if err != nil { log.Warnln(err) return err } fmt.Printf("Downloading %v...\n", req.URL()) resp := client.Do(req) if resp.HTTPResponse == nil { log.Warnln(err) return err } fmt.Printf(" %v\n", resp.HTTPResponse.Status) t := time.NewTicker(500 * time.Millisecond) defer t.Stop() LoopRel: for { select { case <-t.C: fmt.Printf(" transferred %v / %v bytes (%.2f%%)\n", resp.BytesComplete(), resp.Size, 100*resp.Progress()) case <-resp.Done: break LoopRel } } if err := resp.Err(); err != nil { log.Warnln("Download failed:", err, feedURL) return err } fmt.Printf("Download saved to %v \n", resp.Filename) localCatalogCsv = resp.Filename } inputFile, err := os.Open(localCatalogCsv) if err != nil { log.Warnln(err) return err } defer inputFile.Close() detect := detector.New() delimiters := detect.DetectDelimiter(inputFile, '"') if len(delimiters) == 0 || len(delimiters) > 1 { return errors.New("No delimiter detected") } cmp.Separator = delimiters[0] cols, err := getHeaders(localCatalogCsv, delimiters[0]) if err != nil { log.Warnln(err) return err } cmp.Fields = cols cmp.Mapping.Update = true for _, col := range cols { switch col { case "link": cmp.Mapping.Product.Redirect = col case "title": cmp.Mapping.Product.Name = col case "id": cmp.Mapping.Product.Reference = col case "gtin": cmp.Mapping.Product.Ean13 = col case "sku", "numero_modele_produit": cmp.Mapping.Product.Sku = col case "price": cmp.Mapping.Product.Price = col case "mpn": cmp.Mapping.Product.Mpn = col case "description": cmp.Mapping.Product.Description = col case "stock": cmp.Mapping.Product.Quantity = col case "image_link": cmp.Mapping.Product.Image = col case "shipping_height": cmp.Mapping.Product.Height = col case "shipping_width": cmp.Mapping.Product.Width = col case "shipping_weight": cmp.Mapping.Product.Weight = col case "shipping_cost": cmp.Mapping.Product.Shipping = col case "age_group", "availability", "availability_date", "brand", "color", "condition", "currency", "ecotax", "energy_efficiency_class", "gender", "is_bundle", "material", "multipack", "pattern", "pneu_charge", "pneu_diametre", "pneu_fueleconomy", "pneu_hauteur", "pneu_largeur", "pneu_modele", "pneu_noisedb", "pneu_noiselevel", "pneu_rechape", "pneu_renforce", "pneu_runflat", "pneu_saison", "pneu_vitesse", "pneu_wetadhesion", "shipping", "shipping_length", "shipping_time", "size", "size_system", "size_type", "style", "warranty": cmp.Mapping.Product.Features = append(cmp.Mapping.Product.Features, col) case "category": cmp.Mapping.Category.Breadcrumb = col } } cmp, total, err := csv2ps(db, localCatalogCsv, localCatalogFormattedCsv, cols, delimiters[0], cmp) checkErr("while writing formatted for prestashop csv file", err) cmp.Total = total gtotal += total outputBytes, err := csv2json(localCatalogCsv, cols, delimiters[0]) if err != nil { log.Warnln(err) return err } cmpBytes, err := yaml.Marshal(cmp) if err != nil { fmt.Printf("err: %v\n", err) return err } err = ioutil.WriteFile(localCatalogMapYaml, cmpBytes, 0644) checkErr("while writing mapping yaml file", err) err = ioutil.WriteFile(localCatalogJson, outputBytes, 0644) checkErr("while writing json file", err) json2struct.SetDebug(options.debug) opt := json2struct.Options{ UseOmitempty: false, UseShortStruct: true, UseLocal: false, UseExample: false, Prefix: "", Suffix: "", Name: strings.ToLower(feedName), } jsonFile, err := os.Open(localCatalogJson) if err != nil { log.Warnln(err) return err } defer jsonFile.Close() parsed, err := json2struct.Parse(jsonFile, opt) if err != nil { return err } csvRowLine := 1 csvRowExample, err := getExampleRow(localCatalogCsv, delimiters[0], cols, csvRowLine) if err != nil { return err } structName := reStruct.FindStringSubmatch(parsed) parsed = reTags.ReplaceAllString(parsed, `json:"$1" struct2map:"key:$1"`) catalogs = append(catalogs, structName[1]) structResult := bytes.NewBufferString("") structTemplate, _ := template.New("").Parse(packageTemplate) structTemplate.Execute(structResult, map[string]string{ "CatalogSeparator": delimiters[0], "CatalogPath": localCatalogCsv, "Line": fmt.Sprintf("%d", csvRowLine), "Struct": parsed, "StructName": stripCtlAndExtFromUnicode(structName[1]), "Row": strings.Join(csvRowExample, ",\n")}, ) err = ioutil.WriteFile(localCatalogStruct, structResult.Bytes(), 0644) if err != nil { return err } return nil }(feed) t.Throttle() } if t.Err() != nil { for i, err := range t.Errs() { log.Printf("error #%d: %s", i, err) } log.Fatal(t.Err()) } localCatalogBase := filepath.Join("..", "..", "internal", "netaffiliation", fmt.Sprintf("%s.go", "base")) catalogResult := bytes.NewBufferString("") catalogTemplate, _ := template.New("").Parse(baseTemplate) catalogTemplate.Execute(catalogResult, map[string][]string{"Catalogs": catalogs}) err = ioutil.WriteFile(localCatalogBase, catalogResult.Bytes(), 0644) checkErr("while writing struct file", err) pp.Println("total new entries:", gtotal) }, }
View Source
var RootCmd = &cobra.Command{
Use: "effi2ps",
Short: "effi2ps is an helper to load effiliation product feeds into a prestashop database.",
Long: `effi2ps is an helper to load effiliation product feeds into a prestashop database.`,
}
RootCmd is the root command for ovh-qa
Functions ¶
func Execute ¶
func Execute()
Execute adds all child commands to the root command and sets flags appropriately.
func ReplaceSoloCarriageReturns ¶
ReplaceSoloCarriageReturns wraps an io.Reader, on every call of Read it for instances of lonely \r replacing them with \r\n before returning to the end customer lots of files in the wild will come without "proper" line breaks, which irritates go's standard csv package. This'll fix by wrapping the reader passed to csv.NewReader:
rdr, err := csv.NewReader(ReplaceSoloCarriageReturns(r))
Types ¶
type CatalogMap ¶
type CatalogMap struct { Name string `yaml:"name"` Feed string `yaml:"feed"` SellerId int `yaml:"seller_id"` Separator string `yaml:"separator"` Fields []string `yaml:"fields"` Total int `yaml:"total"` Mapping struct { Update bool `yaml:"update" default:"true"` // Either ADD LangSuffix []string `yaml:"multi_language_suffix"` LangFields []string `yaml:"multi_language_fields"` Product Product `yaml:"product"` Category struct { Name string `yaml:"name"` Breadcrumb string `yaml:"breaddcrumb"` Separator string `yaml:"separator"` } `yaml:"category"` } `yaml:"mapping"` }
type Product ¶
type Product struct { Name string `yaml:"name"` Reference string `yaml:"reference"` Ean13 string `yaml:"ean13"` Sku string `yaml:"sku"` Mpn string `yaml:"mpn"` Price string `yaml:"price"` Description string `yaml:"description"` DescriptionShort string `yaml:"description_short"` Image string `yaml:"image"` Quantity string `yaml:"quantity"` Width string `yaml:"width"` Height string `yaml:"height"` Weight string `yaml:"weight"` Shipping string `yaml:"shipping"` Redirect string `yaml:"redirect"` Attributes []string `yaml:"attributes"` Features []string `yaml:"features"` }
type WkMpSeller ¶
type WkMpSeller struct { IdSeller int ShopNameUnique string LinkRewrite string SellerFirstname string SellerLastname string BusinessEmail string Phone string Fax string Address string Postcode string City string IdCountry int IdState int TaxIdentificationNumber string DefaultLang int FacebookId string TwitterId string GoogleId string InstagramId string ProfileImage string ProfileBanner string ShopImage string ShopBanner string Active bool ShopApproved bool SellerCustomerId int SellerDetailsAccess string DateAdd time.Time DateUpd time.Time }
Click to show internal directories.
Click to hide internal directories.